Removed all files from the repository

LICENSE.txt

@@ -1,437 +0,0 @@
-Attribution-NonCommercial-ShareAlike 4.0 International
-
-=======================================================================
-
-Creative Commons Corporation ("Creative Commons") is not a law firm and
-does not provide legal services or legal advice. Distribution of
-Creative Commons public licenses does not create a lawyer-client or
-other relationship. Creative Commons makes its licenses and related
-information available on an "as-is" basis. Creative Commons gives no
-warranties regarding its licenses, any material licensed under their
-terms and conditions, or any related information. Creative Commons
-disclaims all liability for damages resulting from their use to the
-fullest extent possible.
-
-Using Creative Commons Public Licenses
-
-Creative Commons public licenses provide a standard set of terms and
-conditions that creators and other rights holders may use to share
-original works of authorship and other material subject to copyright
-and certain other rights specified in the public license below. The
-following considerations are for informational purposes only, are not
-exhaustive, and do not form part of our licenses.
-
-     Considerations for licensors: Our public licenses are
-     intended for use by those authorized to give the public
-     permission to use material in ways otherwise restricted by
-     copyright and certain other rights. Our licenses are
-     irrevocable. Licensors should read and understand the terms
-     and conditions of the license they choose before applying it.
-     Licensors should also secure all rights necessary before
-     applying our licenses so that the public can reuse the
-     material as expected. Licensors should clearly mark any
-     material not subject to the license. This includes other CC-
-     licensed material, or material used under an exception or
-     limitation to copyright. More considerations for licensors:
-    wiki.creativecommons.org/Considerations_for_licensors
-
-     Considerations for the public: By using one of our public
-     licenses, a licensor grants the public permission to use the
-     licensed material under specified terms and conditions. If
-     the licensor's permission is not necessary for any reason--for
-     example, because of any applicable exception or limitation to
-     copyright--then that use is not regulated by the license. Our
-     licenses grant only permissions under copyright and certain
-     other rights that a licensor has authority to grant. Use of
-     the licensed material may still be restricted for other
-     reasons, including because others have copyright or other
-     rights in the material. A licensor may make special requests,
-     such as asking that all changes be marked or described.
-     Although not required by our licenses, you are encouraged to
-     respect those requests where reasonable. More considerations
-     for the public:
-    wiki.creativecommons.org/Considerations_for_licensees
-
-=======================================================================
-
-Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International
-Public License
-
-By exercising the Licensed Rights (defined below), You accept and agree
-to be bound by the terms and conditions of this Creative Commons
-Attribution-NonCommercial-ShareAlike 4.0 International Public License
-("Public License"). To the extent this Public License may be
-interpreted as a contract, You are granted the Licensed Rights in
-consideration of Your acceptance of these terms and conditions, and the
-Licensor grants You such rights in consideration of benefits the
-Licensor receives from making the Licensed Material available under
-these terms and conditions.
-
-
-Section 1 -- Definitions.
-
-  a. Adapted Material means material subject to Copyright and Similar
-     Rights that is derived from or based upon the Licensed Material
-     and in which the Licensed Material is translated, altered,
-     arranged, transformed, or otherwise modified in a manner requiring
-     permission under the Copyright and Similar Rights held by the
-     Licensor. For purposes of this Public License, where the Licensed
-     Material is a musical work, performance, or sound recording,
-     Adapted Material is always produced where the Licensed Material is
-     synched in timed relation with a moving image.
-
-  b. Adapter's License means the license You apply to Your Copyright
-     and Similar Rights in Your contributions to Adapted Material in
-     accordance with the terms and conditions of this Public License.
-
-  c. BY-NC-SA Compatible License means a license listed at
-     creativecommons.org/compatiblelicenses, approved by Creative
-     Commons as essentially the equivalent of this Public License.
-
-  d. Copyright and Similar Rights means copyright and/or similar rights
-     closely related to copyright including, without limitation,
-     performance, broadcast, sound recording, and Sui Generis Database
-     Rights, without regard to how the rights are labeled or
-     categorized. For purposes of this Public License, the rights
-     specified in Section 2(b)(1)-(2) are not Copyright and Similar
-     Rights.
-
-  e. Effective Technological Measures means those measures that, in the
-     absence of proper authority, may not be circumvented under laws
-     fulfilling obligations under Article 11 of the WIPO Copyright
-     Treaty adopted on December 20, 1996, and/or similar international
-     agreements.
-
-  f. Exceptions and Limitations means fair use, fair dealing, and/or
-     any other exception or limitation to Copyright and Similar Rights
-     that applies to Your use of the Licensed Material.
-
-  g. License Elements means the license attributes listed in the name
-     of a Creative Commons Public License. The License Elements of this
-     Public License are Attribution, NonCommercial, and ShareAlike.
-
-  h. Licensed Material means the artistic or literary work, database,
-     or other material to which the Licensor applied this Public
-     License.
-
-  i. Licensed Rights means the rights granted to You subject to the
-     terms and conditions of this Public License, which are limited to
-     all Copyright and Similar Rights that apply to Your use of the
-     Licensed Material and that the Licensor has authority to license.
-
-  j. Licensor means the individual(s) or entity(ies) granting rights
-     under this Public License.
-
-  k. NonCommercial means not primarily intended for or directed towards
-     commercial advantage or monetary compensation. For purposes of
-     this Public License, the exchange of the Licensed Material for
-     other material subject to Copyright and Similar Rights by digital
-     file-sharing or similar means is NonCommercial provided there is
-     no payment of monetary compensation in connection with the
-     exchange.
-
-  l. Share means to provide material to the public by any means or
-     process that requires permission under the Licensed Rights, such
-     as reproduction, public display, public performance, distribution,
-     dissemination, communication, or importation, and to make material
-     available to the public including in ways that members of the
-     public may access the material from a place and at a time
-     individually chosen by them.
-
-  m. Sui Generis Database Rights means rights other than copyright
-     resulting from Directive 96/9/EC of the European Parliament and of
-     the Council of 11 March 1996 on the legal protection of databases,
-     as amended and/or succeeded, as well as other essentially
-     equivalent rights anywhere in the world.
-
-  n. You means the individual or entity exercising the Licensed Rights
-     under this Public License. Your has a corresponding meaning.
-
-
-Section 2 -- Scope.
-
-  a. License grant.
-
-       1. Subject to the terms and conditions of this Public License,
-          the Licensor hereby grants You a worldwide, royalty-free,
-          non-sublicensable, non-exclusive, irrevocable license to
-          exercise the Licensed Rights in the Licensed Material to:
-
-            a. reproduce and Share the Licensed Material, in whole or
-               in part, for NonCommercial purposes only; and
-
-            b. produce, reproduce, and Share Adapted Material for
-               NonCommercial purposes only.
-
-       2. Exceptions and Limitations. For the avoidance of doubt, where
-          Exceptions and Limitations apply to Your use, this Public
-          License does not apply, and You do not need to comply with
-          its terms and conditions.
-
-       3. Term. The term of this Public License is specified in Section
-          6(a).
-
-       4. Media and formats; technical modifications allowed. The
-          Licensor authorizes You to exercise the Licensed Rights in
-          all media and formats whether now known or hereafter created,
-          and to make technical modifications necessary to do so. The
-          Licensor waives and/or agrees not to assert any right or
-          authority to forbid You from making technical modifications
-          necessary to exercise the Licensed Rights, including
-          technical modifications necessary to circumvent Effective
-          Technological Measures. For purposes of this Public License,
-          simply making modifications authorized by this Section 2(a)
-          (4) never produces Adapted Material.
-
-       5. Downstream recipients.
-
-            a. Offer from the Licensor -- Licensed Material. Every
-               recipient of the Licensed Material automatically
-               receives an offer from the Licensor to exercise the
-               Licensed Rights under the terms and conditions of this
-               Public License.
-
-            b. Additional offer from the Licensor -- Adapted Material.
-               Every recipient of Adapted Material from You
-               automatically receives an offer from the Licensor to
-               exercise the Licensed Rights in the Adapted Material
-               under the conditions of the Adapter's License You apply.
-
-            c. No downstream restrictions. You may not offer or impose
-               any additional or different terms or conditions on, or
-               apply any Effective Technological Measures to, the
-               Licensed Material if doing so restricts exercise of the
-               Licensed Rights by any recipient of the Licensed
-               Material.
-
-       6. No endorsement. Nothing in this Public License constitutes or
-          may be construed as permission to assert or imply that You
-          are, or that Your use of the Licensed Material is, connected
-          with, or sponsored, endorsed, or granted official status by,
-          the Licensor or others designated to receive attribution as
-          provided in Section 3(a)(1)(A)(i).
-
-  b. Other rights.
-
-       1. Moral rights, such as the right of integrity, are not
-          licensed under this Public License, nor are publicity,
-          privacy, and/or other similar personality rights; however, to
-          the extent possible, the Licensor waives and/or agrees not to
-          assert any such rights held by the Licensor to the limited
-          extent necessary to allow You to exercise the Licensed
-          Rights, but not otherwise.
-
-       2. Patent and trademark rights are not licensed under this
-          Public License.
-
-       3. To the extent possible, the Licensor waives any right to
-          collect royalties from You for the exercise of the Licensed
-          Rights, whether directly or through a collecting society
-          under any voluntary or waivable statutory or compulsory
-          licensing scheme. In all other cases the Licensor expressly
-          reserves any right to collect such royalties, including when
-          the Licensed Material is used other than for NonCommercial
-          purposes.
-
-
-Section 3 -- License Conditions.
-
-Your exercise of the Licensed Rights is expressly made subject to the
-following conditions.
-
-  a. Attribution.
-
-       1. If You Share the Licensed Material (including in modified
-          form), You must:
-
-            a. retain the following if it is supplied by the Licensor
-               with the Licensed Material:
-
-                 i. identification of the creator(s) of the Licensed
-                    Material and any others designated to receive
-                    attribution, in any reasonable manner requested by
-                    the Licensor (including by pseudonym if
-                    designated);
-
-                ii. a copyright notice;
-
-               iii. a notice that refers to this Public License;
-
-                iv. a notice that refers to the disclaimer of
-                    warranties;
-
-                 v. a URI or hyperlink to the Licensed Material to the
-                    extent reasonably practicable;
-
-            b. indicate if You modified the Licensed Material and
-               retain an indication of any previous modifications; and
-
-            c. indicate the Licensed Material is licensed under this
-               Public License, and include the text of, or the URI or
-               hyperlink to, this Public License.
-
-       2. You may satisfy the conditions in Section 3(a)(1) in any
-          reasonable manner based on the medium, means, and context in
-          which You Share the Licensed Material. For example, it may be
-          reasonable to satisfy the conditions by providing a URI or
-          hyperlink to a resource that includes the required
-          information.
-       3. If requested by the Licensor, You must remove any of the
-          information required by Section 3(a)(1)(A) to the extent
-          reasonably practicable.
-
-  b. ShareAlike.
-
-     In addition to the conditions in Section 3(a), if You Share
-     Adapted Material You produce, the following conditions also apply.
-
-       1. The Adapter's License You apply must be a Creative Commons
-          license with the same License Elements, this version or
-          later, or a BY-NC-SA Compatible License.
-
-       2. You must include the text of, or the URI or hyperlink to, the
-          Adapter's License You apply. You may satisfy this condition
-          in any reasonable manner based on the medium, means, and
-          context in which You Share Adapted Material.
-
-       3. You may not offer or impose any additional or different terms
-          or conditions on, or apply any Effective Technological
-          Measures to, Adapted Material that restrict exercise of the
-          rights granted under the Adapter's License You apply.
-
-
-Section 4 -- Sui Generis Database Rights.
-
-Where the Licensed Rights include Sui Generis Database Rights that
-apply to Your use of the Licensed Material:
-
-  a. for the avoidance of doubt, Section 2(a)(1) grants You the right
-     to extract, reuse, reproduce, and Share all or a substantial
-     portion of the contents of the database for NonCommercial purposes
-     only;
-
-  b. if You include all or a substantial portion of the database
-     contents in a database in which You have Sui Generis Database
-     Rights, then the database in which You have Sui Generis Database
-     Rights (but not its individual contents) is Adapted Material,
-     including for purposes of Section 3(b); and
-
-  c. You must comply with the conditions in Section 3(a) if You Share
-     all or a substantial portion of the contents of the database.
-
-For the avoidance of doubt, this Section 4 supplements and does not
-replace Your obligations under this Public License where the Licensed
-Rights include other Copyright and Similar Rights.
-
-
-Section 5 -- Disclaimer of Warranties and Limitation of Liability.
-
-  a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE
-     EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS
-     AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF
-     ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS,
-     IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION,
-     WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR
-     PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS,
-     ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT
-     KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT
-     ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU.
-
-  b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE
-     TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION,
-     NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT,
-     INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES,
-     COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR
-     USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN
-     ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR
-     DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR
-     IN PART, THIS LIMITATION MAY NOT APPLY TO YOU.
-
-  c. The disclaimer of warranties and limitation of liability provided
-     above shall be interpreted in a manner that, to the extent
-     possible, most closely approximates an absolute disclaimer and
-     waiver of all liability.
-
-
-Section 6 -- Term and Termination.
-
-  a. This Public License applies for the term of the Copyright and
-     Similar Rights licensed here. However, if You fail to comply with
-     this Public License, then Your rights under this Public License
-     terminate automatically.
-
-  b. Where Your right to use the Licensed Material has terminated under
-     Section 6(a), it reinstates:
-
-       1. automatically as of the date the violation is cured, provided
-          it is cured within 30 days of Your discovery of the
-          violation; or
-
-       2. upon express reinstatement by the Licensor.
-
-     For the avoidance of doubt, this Section 6(b) does not affect any
-     right the Licensor may have to seek remedies for Your violations
-     of this Public License.
-
-  c. For the avoidance of doubt, the Licensor may also offer the
-     Licensed Material under separate terms or conditions or stop
-     distributing the Licensed Material at any time; however, doing so
-     will not terminate this Public License.
-
-  d. Sections 1, 5, 6, 7, and 8 survive termination of this Public
-     License.
-
-
-Section 7 -- Other Terms and Conditions.
-
-  a. The Licensor shall not be bound by any additional or different
-     terms or conditions communicated by You unless expressly agreed.
-
-  b. Any arrangements, understandings, or agreements regarding the
-     Licensed Material not stated herein are separate from and
-     independent of the terms and conditions of this Public License.
-
-
-Section 8 -- Interpretation.
-
-  a. For the avoidance of doubt, this Public License does not, and
-     shall not be interpreted to, reduce, limit, restrict, or impose
-     conditions on any use of the Licensed Material that could lawfully
-     be made without permission under this Public License.
-
-  b. To the extent possible, if any provision of this Public License is
-     deemed unenforceable, it shall be automatically reformed to the
-     minimum extent necessary to make it enforceable. If the provision
-     cannot be reformed, it shall be severed from this Public License
-     without affecting the enforceability of the remaining terms and
-     conditions.
-
-  c. No term or condition of this Public License will be waived and no
-     failure to comply consented to unless expressly agreed to by the
-     Licensor.
-
-  d. Nothing in this Public License constitutes or may be interpreted
-     as a limitation upon, or waiver of, any privileges and immunities
-     that apply to the Licensor or You, including from the legal
-     processes of any jurisdiction or authority.
-
-=======================================================================
-
-Creative Commons is not a party to its public
-licenses. Notwithstanding, Creative Commons may elect to apply one of
-its public licenses to material it publishes and in those instances
-will be considered the “Licensor.” The text of the Creative Commons
-public licenses is dedicated to the public domain under the CC0 Public
-Domain Dedication. Except for the limited purpose of indicating that
-material is shared under a Creative Commons public license or as
-otherwise permitted by the Creative Commons policies published at
-creativecommons.org/policies, Creative Commons does not authorize the
-use of the trademark "Creative Commons" or any other trademark or logo
-of Creative Commons without its prior written consent including,
-without limitation, in connection with any unauthorized modifications
-to any of its public licenses or any other arrangements,
-understandings, or agreements concerning use of licensed material. For
-the avoidance of doubt, this paragraph does not form part of the
-public licenses.
-
-Creative Commons may be contacted at creativecommons.org.

README.md

@@ -1,68 +0,0 @@
----
-language:
-- en
-- ar
-metrics:
-- accuracy
-pipeline_tag: text-generation
-tags:
-- medical
-license: cc-by-nc-sa-4.0
----
-
-## Model Card for BiMediX-Bilingual
-
-### Model Details
-- **Name:** BiMediX
-- **Version:** 1.0
-- **Type:** Bilingual Medical Mixture of Experts Large Language Model (LLM)
-- **Languages:** English, Arabic
-- **Model Architecture:** [Mixtral-8x7B-Instruct-v0.1](https://huggingface.co/mistralai/Mixtral-8x7B-Instruct-v0.1)
-- **Training Data:** BiMed1.3M, a bilingual dataset with diverse medical interactions.
-
-### Intended Use
-- **Primary Use:** Medical interactions in both English and Arabic.
-- **Capabilities:** MCQA, closed QA and chats.
-
-## Getting Started
-
-```python
-from transformers import AutoModelForCausalLM, AutoTokenizer
-
-model_id = "BiMediX/BiMediX-Bi"
-
-tokenizer = AutoTokenizer.from_pretrained(model_id)
-model = AutoModelForCausalLM.from_pretrained(model_id)
-
-text = "Hello BiMediX! I've been experiencing increased tiredness in the past week."
-inputs = tokenizer(text, return_tensors="pt")
-
-outputs = model.generate(**inputs, max_new_tokens=500)
-print(tokenizer.decode(outputs[0], skip_special_tokens=True))
-```
-
-### Training Procedure
-- **Dataset:** BiMed1.3M, 632 million healthcare specialized tokens.
-- **QLoRA Adaptation:** Implements a low-rank adaptation technique, incorporating learnable low-rank adapter weights into the experts and the routing network. This results in training about 4% of the original parameters.
-- **Training Resources:** The model underwent training on approximately 632 million tokens from the Arabic-English corpus, including 288 million tokens exclusively for English.
-
-### Model Performance
-- **Benchmarks:** Outperforms the baseline model and Jais-30B in medical evaluations.
-
-| **Model**                         | **CKG** | **CBio** | **CMed** | **MedGen** | **ProMed** | **Ana** | **MedMCQA** | **MedQA** | **PubmedQA** | **AVG** |
-|-----------------------------------|------------|-----------|-----------|-------------|-------------|---------|-------------|-----------|--------------|---------|
-| Jais-30B | 57.4       | 55.2      | 46.2      | 55.0        | 46.0        | 48.9    | 40.2        | 31.0      | 75.5         | 50.6    |
-| Mixtral-8x7B| 59.1       | 57.6      | 52.6      | 59.5        | 53.3        | 54.4    | 43.2        | 40.6      | 74.7         | 55.0    |
-| **BiMediX (Bilingual)**           | **70.6**   | **72.2**  | **59.3**  | **74.0**    | **64.2**    | **59.6**| **55.8**    | **54.0**  | **78.6**     | **65.4**|
-
-### Safety and Ethical Considerations
-- **Potential issues**: hallucinations, toxicity, stereotypes.
-- **Usage:** Research purposes only.
-
-### Accessibility
-- **Availability:** [BiMediX GitHub Repository](https://github.com/mbzuai-oryx/BiMediX).
-- arxiv.org/abs/2402.13253 
-
-### Authors
-Sara Pieri, Sahal Shaji Mullappilly, Fahad Shahbaz Khan, Rao Muhammad Anwer Salman Khan, Timothy Baldwin, Hisham Cholakkal  
-**Mohamed Bin Zayed University of Artificial Intelligence (MBZUAI)**

config.json

@@ -1,30 +0,0 @@
-{
-  "_name_or_path": "BiMediX_ar_en",
-  "architectures": [
-    "MixtralModel"
-  ],
-  "attention_dropout": 0.0,
-  "bos_token_id": 1,
-  "eos_token_id": 2,
-  "hidden_act": "silu",
-  "hidden_size": 4096,
-  "initializer_range": 0.02,
-  "intermediate_size": 14336,
-  "max_position_embeddings": 32768,
-  "model_type": "mixtral",
-  "num_attention_heads": 32,
-  "num_experts_per_tok": 2,
-  "num_hidden_layers": 32,
-  "num_key_value_heads": 8,
-  "num_local_experts": 8,
-  "output_router_logits": true,
-  "rms_norm_eps": 1e-05,
-  "rope_theta": 1000000.0,
-  "router_aux_loss_coef": 0.02,
-  "sliding_window": null,
-  "tie_word_embeddings": false,
-  "torch_dtype": "float32",
-  "transformers_version": "4.36.2",
-  "use_cache": false,
-  "vocab_size": 32000
-}

model-00001-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:1baf6fc0c23c3a7fc7536b242ed6991f91d42c21af13565e2fa20903756f2dc5
-size 4920052576

model-00002-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:810862c8a4917d2796787a62b837bebfade62c42f7683a56017267fd47878677
-size 4865559752

model-00003-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:79fd765378879f4a840c31198fd14c99c7a8cb62cb1c931eb8d95aee5b979495
-size 4865559752

model-00004-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:2fcf66b0d6278f7aba3b84795728096705ca27394fa85b8bda0909283c806083
-size 4865559752

model-00005-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:71418f4cfe7ff1ff318f952ec9c22d96f6366160a25fa96f4449670beb6cde2f
-size 4865559752

model-00006-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:b4cea7fe3b6dd5cbbaa2a56b78b2f7f9b8fd0b645240bec94b7162ef564ab01f
-size 4932504136

model-00007-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:d00d477df028778f60e272601c2ba43f94561751e2a717219c47ec4fed7e0a01
-size 4865559752

model-00008-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:309de74e6574a94901c55a95122e767afd3563542fc896c4edd691dcd208e514
-size 4865559752

model-00009-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:6ac577bb5e6eaf934a4a01287ec38178134fb73f1a999599f615f99d1bea9317
-size 4865559752

model-00010-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:51c4495807859a642892301ada42b1587113271b9f508deaf6458a9d5a01d647
-size 4865559752

model-00011-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:47cd7fcd7f3d69a7734e6f26f0b44310c942edeea9d085e1b75cb55276f96c95
-size 4865559752

model-00012-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:31157c892a1aa6d10abd7ad2b1c93b9eb20ba0668c55fb0f36b599bb58e4ade7
-size 4999646096

model-00013-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:9e2cb4e851b4d2df44b983c2763ce7b28d3cb2ef768018fb0b2ad99a2713b84b
-size 4798417824

model-00014-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:30a731ee2585039601ee2759cd6c80999533a7d377d5586110452c35c56c8a5f
-size 4865559776

model-00015-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:780505df2dfa8e17130e485744ebacf22ce1a2a02aa3d9164b69f66983b9af5b
-size 4865559784

model-00016-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:b4ab5ad113b2554f96269626eca3c44b1d0ee8ae7253544c0959f61a586300f5
-size 4865559784

model-00017-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:db2a84fc5bfcdb482ffc6d8e4e27de2b087eecbb8ef996cdb38c9fd3375ccf16
-size 4865559784

model-00018-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:3e179e90f8650fb8ab5007f350c03c72a7004df7f981ccadb63b4622a1aa6e8a
-size 4865559784

model-00019-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:3f3439ff47e2a0090f382331d1047f95d07069e6c5ffd73c2f2fb16e659a99f7
-size 4932504152

model-00020-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:1138eb26d7fd872cde0c3000f8d91d15ee315e39cf3398c0de8fd1b75f4abf50
-size 4865559776

model-00021-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:ceeff72624282350a2df47390e61d36ae983b89ac5eadb5b3faf5791060627b0
-size 4865559784

model-00022-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:a78e7ee661fb1201c88c4523824e344d8e0f43b551b08ae13d82e3750d1ed0a2
-size 4865559784

model-00023-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:57be616a1c580c580075daa2da23b347387675437dafe352ac9a13cb53a6834e
-size 4865559784

model-00024-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:f1f7904f55f37b96c604018ed4ea6ee540753ad7de1c825105d419373087d55b
-size 4865559784

model-00025-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:d63299a83cad8c1775d977653f001fafa4be8aa11abae8817ae44fc9f048957c
-size 4932504152

model-00026-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:6268c4c42bde8dfabbe8f5080459741c1885ab41b147a0c176aa59c1b6b16a9c
-size 4865559776

model-00027-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:22a7707d48b52c016a9984eca0cabeec487218415f68bfede26ad4fa81567832
-size 4865559784

model-00028-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:34480602749cfef8703153d159e4f36f7265be3e17f112682ed5172c10baa670
-size 4865559784

model-00029-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:dff36462f51ef5a2c535cf6f2e5a05ff99b144bdd51c1e88b52ea86720c91802
-size 4865559784

model-00030-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:f38fb4c2c9d6ec9fd45a92ef8d5d187f8cb138c13a9af35e049cbefa241cc129
-size 4865559784

model-00031-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:cdf152377e648ccc76f2e77683f0f2e7f989d3f5a7a32912965313021a5eccf2
-size 4932504152

model-00032-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:6e840d3f632f1822ad825e3346f6b008cc61df2ccad52d965004be70991ff03f
-size 4865559776

model-00033-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:f6a549ae04c02ca76480b7484e0edc69f2bd9f0bc2170ffb4b6b3a0ff53cfe36
-size 4865559784

model-00034-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:2dec50307f73828911ab1c410d5024aec45fbaeeadcf1533a6e339b1689ed2f3
-size 4865559784

model-00035-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:ea57002c4bb181021881f374cbdaae5b96a28b1a88fb9db59dc62e6fb32c7a9d
-size 4865559784

model-00036-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:8ac0f9a6e881e4495948a77041cbe6cde961f67d4bb144d9d80d2d564e412fe2
-size 4865559784

model-00037-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:35af04df3f45ec69786da79d60dfcd71aebc1e5f3b2587e50bbb83d431e11c02
-size 4999646120

model-00038-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:a02195d61f5a74711097b962e1b31c750319774a7a33bda2afbd11d92d6b8b6e
-size 4798417824

model-00039-of-00039.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:b39804dc082c5ba7ae8d6e1dce7a574c5df482eb2929a21531e38489a7c1a549
-size 939574072

model.safetensors.index.json

@@ -1,1001 +0,0 @@
-{
-  "metadata": {
-    "total_size": 186286882816
-  },
-  "weight_map": {
-    "embed_tokens.weight": "model-00001-of-00039.safetensors",
-    "layers.0.block_sparse_moe.experts.0.w1.weight": "model-00001-of-00039.safetensors",
-    "layers.0.block_sparse_moe.experts.0.w2.weight": "model-00001-of-00039.safetensors",
-    "layers.0.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00039.safetensors",
-    "layers.0.block_sparse_moe.experts.1.w1.weight": "model-00001-of-00039.safetensors",
-    "layers.0.block_sparse_moe.experts.1.w2.weight": "model-00001-of-00039.safetensors",
-    "layers.0.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00039.safetensors",
-    "layers.0.block_sparse_moe.experts.2.w1.weight": "model-00001-of-00039.safetensors",
-    "layers.0.block_sparse_moe.experts.2.w2.weight": "model-00001-of-00039.safetensors",
-    "layers.0.block_sparse_moe.experts.2.w3.weight": "model-00001-of-00039.safetensors",
-    "layers.0.block_sparse_moe.experts.3.w1.weight": "model-00001-of-00039.safetensors",
-    "layers.0.block_sparse_moe.experts.3.w2.weight": "model-00001-of-00039.safetensors",
-    "layers.0.block_sparse_moe.experts.3.w3.weight": "model-00001-of-00039.safetensors",
-    "layers.0.block_sparse_moe.experts.4.w1.weight": "model-00001-of-00039.safetensors",
-    "layers.0.block_sparse_moe.experts.4.w2.weight": "model-00001-of-00039.safetensors",
-    "layers.0.block_sparse_moe.experts.4.w3.weight": "model-00001-of-00039.safetensors",
-    "layers.0.block_sparse_moe.experts.5.w1.weight": "model-00001-of-00039.safetensors",
-    "layers.0.block_sparse_moe.experts.5.w2.weight": "model-00001-of-00039.safetensors",
-    "layers.0.block_sparse_moe.experts.5.w3.weight": "model-00001-of-00039.safetensors",
-    "layers.0.block_sparse_moe.experts.6.w1.weight": "model-00002-of-00039.safetensors",
-    "layers.0.block_sparse_moe.experts.6.w2.weight": "model-00002-of-00039.safetensors",
-    "layers.0.block_sparse_moe.experts.6.w3.weight": "model-00002-of-00039.safetensors",
-    "layers.0.block_sparse_moe.experts.7.w1.weight": "model-00002-of-00039.safetensors",
-    "layers.0.block_sparse_moe.experts.7.w2.weight": "model-00002-of-00039.safetensors",
-    "layers.0.block_sparse_moe.experts.7.w3.weight": "model-00002-of-00039.safetensors",
-    "layers.0.block_sparse_moe.gate.weight": "model-00001-of-00039.safetensors",
-    "layers.0.input_layernorm.weight": "model-00002-of-00039.safetensors",
-    "layers.0.post_attention_layernorm.weight": "model-00002-of-00039.safetensors",
-    "layers.0.self_attn.k_proj.weight": "model-00001-of-00039.safetensors",
-    "layers.0.self_attn.o_proj.weight": "model-00001-of-00039.safetensors",
-    "layers.0.self_attn.q_proj.weight": "model-00001-of-00039.safetensors",
-    "layers.0.self_attn.v_proj.weight": "model-00001-of-00039.safetensors",
-    "layers.1.block_sparse_moe.experts.0.w1.weight": "model-00002-of-00039.safetensors",
-    "layers.1.block_sparse_moe.experts.0.w2.weight": "model-00002-of-00039.safetensors",
-    "layers.1.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00039.safetensors",
-    "layers.1.block_sparse_moe.experts.1.w1.weight": "model-00002-of-00039.safetensors",
-    "layers.1.block_sparse_moe.experts.1.w2.weight": "model-00002-of-00039.safetensors",
-    "layers.1.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00039.safetensors",
-    "layers.1.block_sparse_moe.experts.2.w1.weight": "model-00002-of-00039.safetensors",
-    "layers.1.block_sparse_moe.experts.2.w2.weight": "model-00002-of-00039.safetensors",
-    "layers.1.block_sparse_moe.experts.2.w3.weight": "model-00002-of-00039.safetensors",
-    "layers.1.block_sparse_moe.experts.3.w1.weight": "model-00002-of-00039.safetensors",
-    "layers.1.block_sparse_moe.experts.3.w2.weight": "model-00002-of-00039.safetensors",
-    "layers.1.block_sparse_moe.experts.3.w3.weight": "model-00002-of-00039.safetensors",
-    "layers.1.block_sparse_moe.experts.4.w1.weight": "model-00002-of-00039.safetensors",
-    "layers.1.block_sparse_moe.experts.4.w2.weight": "model-00002-of-00039.safetensors",
-    "layers.1.block_sparse_moe.experts.4.w3.weight": "model-00003-of-00039.safetensors",
-    "layers.1.block_sparse_moe.experts.5.w1.weight": "model-00003-of-00039.safetensors",
-    "layers.1.block_sparse_moe.experts.5.w2.weight": "model-00003-of-00039.safetensors",
-    "layers.1.block_sparse_moe.experts.5.w3.weight": "model-00003-of-00039.safetensors",
-    "layers.1.block_sparse_moe.experts.6.w1.weight": "model-00003-of-00039.safetensors",
-    "layers.1.block_sparse_moe.experts.6.w2.weight": "model-00003-of-00039.safetensors",
-    "layers.1.block_sparse_moe.experts.6.w3.weight": "model-00003-of-00039.safetensors",
-    "layers.1.block_sparse_moe.experts.7.w1.weight": "model-00003-of-00039.safetensors",
-    "layers.1.block_sparse_moe.experts.7.w2.weight": "model-00003-of-00039.safetensors",
-    "layers.1.block_sparse_moe.experts.7.w3.weight": "model-00003-of-00039.safetensors",
-    "layers.1.block_sparse_moe.gate.weight": "model-00002-of-00039.safetensors",
-    "layers.1.input_layernorm.weight": "model-00003-of-00039.safetensors",
-    "layers.1.post_attention_layernorm.weight": "model-00003-of-00039.safetensors",
-    "layers.1.self_attn.k_proj.weight": "model-00002-of-00039.safetensors",
-    "layers.1.self_attn.o_proj.weight": "model-00002-of-00039.safetensors",
-    "layers.1.self_attn.q_proj.weight": "model-00002-of-00039.safetensors",
-    "layers.1.self_attn.v_proj.weight": "model-00002-of-00039.safetensors",
-    "layers.10.block_sparse_moe.experts.0.w1.weight": "model-00013-of-00039.safetensors",
-    "layers.10.block_sparse_moe.experts.0.w2.weight": "model-00013-of-00039.safetensors",
-    "layers.10.block_sparse_moe.experts.0.w3.weight": "model-00013-of-00039.safetensors",
-    "layers.10.block_sparse_moe.experts.1.w1.weight": "model-00013-of-00039.safetensors",
-    "layers.10.block_sparse_moe.experts.1.w2.weight": "model-00013-of-00039.safetensors",
-    "layers.10.block_sparse_moe.experts.1.w3.weight": "model-00013-of-00039.safetensors",
-    "layers.10.block_sparse_moe.experts.2.w1.weight": "model-00013-of-00039.safetensors",
-    "layers.10.block_sparse_moe.experts.2.w2.weight": "model-00013-of-00039.safetensors",
-    "layers.10.block_sparse_moe.experts.2.w3.weight": "model-00013-of-00039.safetensors",
-    "layers.10.block_sparse_moe.experts.3.w1.weight": "model-00013-of-00039.safetensors",
-    "layers.10.block_sparse_moe.experts.3.w2.weight": "model-00013-of-00039.safetensors",
-    "layers.10.block_sparse_moe.experts.3.w3.weight": "model-00013-of-00039.safetensors",
-    "layers.10.block_sparse_moe.experts.4.w1.weight": "model-00013-of-00039.safetensors",
-    "layers.10.block_sparse_moe.experts.4.w2.weight": "model-00013-of-00039.safetensors",
-    "layers.10.block_sparse_moe.experts.4.w3.weight": "model-00013-of-00039.safetensors",
-    "layers.10.block_sparse_moe.experts.5.w1.weight": "model-00013-of-00039.safetensors",
-    "layers.10.block_sparse_moe.experts.5.w2.weight": "model-00013-of-00039.safetensors",
-    "layers.10.block_sparse_moe.experts.5.w3.weight": "model-00013-of-00039.safetensors",
-    "layers.10.block_sparse_moe.experts.6.w1.weight": "model-00013-of-00039.safetensors",
-    "layers.10.block_sparse_moe.experts.6.w2.weight": "model-00013-of-00039.safetensors",
-    "layers.10.block_sparse_moe.experts.6.w3.weight": "model-00014-of-00039.safetensors",
-    "layers.10.block_sparse_moe.experts.7.w1.weight": "model-00014-of-00039.safetensors",
-    "layers.10.block_sparse_moe.experts.7.w2.weight": "model-00014-of-00039.safetensors",
-    "layers.10.block_sparse_moe.experts.7.w3.weight": "model-00014-of-00039.safetensors",
-    "layers.10.block_sparse_moe.gate.weight": "model-00013-of-00039.safetensors",
-    "layers.10.input_layernorm.weight": "model-00014-of-00039.safetensors",
-    "layers.10.post_attention_layernorm.weight": "model-00014-of-00039.safetensors",
-    "layers.10.self_attn.k_proj.weight": "model-00013-of-00039.safetensors",
-    "layers.10.self_attn.o_proj.weight": "model-00013-of-00039.safetensors",
-    "layers.10.self_attn.q_proj.weight": "model-00012-of-00039.safetensors",
-    "layers.10.self_attn.v_proj.weight": "model-00013-of-00039.safetensors",
-    "layers.11.block_sparse_moe.experts.0.w1.weight": "model-00014-of-00039.safetensors",
-    "layers.11.block_sparse_moe.experts.0.w2.weight": "model-00014-of-00039.safetensors",
-    "layers.11.block_sparse_moe.experts.0.w3.weight": "model-00014-of-00039.safetensors",
-    "layers.11.block_sparse_moe.experts.1.w1.weight": "model-00014-of-00039.safetensors",
-    "layers.11.block_sparse_moe.experts.1.w2.weight": "model-00014-of-00039.safetensors",
-    "layers.11.block_sparse_moe.experts.1.w3.weight": "model-00014-of-00039.safetensors",
-    "layers.11.block_sparse_moe.experts.2.w1.weight": "model-00014-of-00039.safetensors",
-    "layers.11.block_sparse_moe.experts.2.w2.weight": "model-00014-of-00039.safetensors",
-    "layers.11.block_sparse_moe.experts.2.w3.weight": "model-00014-of-00039.safetensors",
-    "layers.11.block_sparse_moe.experts.3.w1.weight": "model-00014-of-00039.safetensors",
-    "layers.11.block_sparse_moe.experts.3.w2.weight": "model-00014-of-00039.safetensors",
-    "layers.11.block_sparse_moe.experts.3.w3.weight": "model-00014-of-00039.safetensors",
-    "layers.11.block_sparse_moe.experts.4.w1.weight": "model-00014-of-00039.safetensors",
-    "layers.11.block_sparse_moe.experts.4.w2.weight": "model-00014-of-00039.safetensors",
-    "layers.11.block_sparse_moe.experts.4.w3.weight": "model-00014-of-00039.safetensors",
-    "layers.11.block_sparse_moe.experts.5.w1.weight": "model-00014-of-00039.safetensors",
-    "layers.11.block_sparse_moe.experts.5.w2.weight": "model-00015-of-00039.safetensors",
-    "layers.11.block_sparse_moe.experts.5.w3.weight": "model-00015-of-00039.safetensors",
-    "layers.11.block_sparse_moe.experts.6.w1.weight": "model-00015-of-00039.safetensors",
-    "layers.11.block_sparse_moe.experts.6.w2.weight": "model-00015-of-00039.safetensors",
-    "layers.11.block_sparse_moe.experts.6.w3.weight": "model-00015-of-00039.safetensors",
-    "layers.11.block_sparse_moe.experts.7.w1.weight": "model-00015-of-00039.safetensors",
-    "layers.11.block_sparse_moe.experts.7.w2.weight": "model-00015-of-00039.safetensors",
-    "layers.11.block_sparse_moe.experts.7.w3.weight": "model-00015-of-00039.safetensors",
-    "layers.11.block_sparse_moe.gate.weight": "model-00014-of-00039.safetensors",
-    "layers.11.input_layernorm.weight": "model-00015-of-00039.safetensors",
-    "layers.11.post_attention_layernorm.weight": "model-00015-of-00039.safetensors",
-    "layers.11.self_attn.k_proj.weight": "model-00014-of-00039.safetensors",
-    "layers.11.self_attn.o_proj.weight": "model-00014-of-00039.safetensors",
-    "layers.11.self_attn.q_proj.weight": "model-00014-of-00039.safetensors",
-    "layers.11.self_attn.v_proj.weight": "model-00014-of-00039.safetensors",
-    "layers.12.block_sparse_moe.experts.0.w1.weight": "model-00015-of-00039.safetensors",
-    "layers.12.block_sparse_moe.experts.0.w2.weight": "model-00015-of-00039.safetensors",
-    "layers.12.block_sparse_moe.experts.0.w3.weight": "model-00015-of-00039.safetensors",
-    "layers.12.block_sparse_moe.experts.1.w1.weight": "model-00015-of-00039.safetensors",
-    "layers.12.block_sparse_moe.experts.1.w2.weight": "model-00015-of-00039.safetensors",
-    "layers.12.block_sparse_moe.experts.1.w3.weight": "model-00015-of-00039.safetensors",
-    "layers.12.block_sparse_moe.experts.2.w1.weight": "model-00015-of-00039.safetensors",
-    "layers.12.block_sparse_moe.experts.2.w2.weight": "model-00015-of-00039.safetensors",
-    "layers.12.block_sparse_moe.experts.2.w3.weight": "model-00015-of-00039.safetensors",
-    "layers.12.block_sparse_moe.experts.3.w1.weight": "model-00015-of-00039.safetensors",
-    "layers.12.block_sparse_moe.experts.3.w2.weight": "model-00015-of-00039.safetensors",
-    "layers.12.block_sparse_moe.experts.3.w3.weight": "model-00015-of-00039.safetensors",
-    "layers.12.block_sparse_moe.experts.4.w1.weight": "model-00016-of-00039.safetensors",
-    "layers.12.block_sparse_moe.experts.4.w2.weight": "model-00016-of-00039.safetensors",
-    "layers.12.block_sparse_moe.experts.4.w3.weight": "model-00016-of-00039.safetensors",
-    "layers.12.block_sparse_moe.experts.5.w1.weight": "model-00016-of-00039.safetensors",
-    "layers.12.block_sparse_moe.experts.5.w2.weight": "model-00016-of-00039.safetensors",
-    "layers.12.block_sparse_moe.experts.5.w3.weight": "model-00016-of-00039.safetensors",
-    "layers.12.block_sparse_moe.experts.6.w1.weight": "model-00016-of-00039.safetensors",
-    "layers.12.block_sparse_moe.experts.6.w2.weight": "model-00016-of-00039.safetensors",
-    "layers.12.block_sparse_moe.experts.6.w3.weight": "model-00016-of-00039.safetensors",
-    "layers.12.block_sparse_moe.experts.7.w1.weight": "model-00016-of-00039.safetensors",
-    "layers.12.block_sparse_moe.experts.7.w2.weight": "model-00016-of-00039.safetensors",
-    "layers.12.block_sparse_moe.experts.7.w3.weight": "model-00016-of-00039.safetensors",
-    "layers.12.block_sparse_moe.gate.weight": "model-00015-of-00039.safetensors",
-    "layers.12.input_layernorm.weight": "model-00016-of-00039.safetensors",
-    "layers.12.post_attention_layernorm.weight": "model-00016-of-00039.safetensors",
-    "layers.12.self_attn.k_proj.weight": "model-00015-of-00039.safetensors",
-    "layers.12.self_attn.o_proj.weight": "model-00015-of-00039.safetensors",
-    "layers.12.self_attn.q_proj.weight": "model-00015-of-00039.safetensors",
-    "layers.12.self_attn.v_proj.weight": "model-00015-of-00039.safetensors",
-    "layers.13.block_sparse_moe.experts.0.w1.weight": "model-00016-of-00039.safetensors",
-    "layers.13.block_sparse_moe.experts.0.w2.weight": "model-00016-of-00039.safetensors",
-    "layers.13.block_sparse_moe.experts.0.w3.weight": "model-00016-of-00039.safetensors",
-    "layers.13.block_sparse_moe.experts.1.w1.weight": "model-00016-of-00039.safetensors",
-    "layers.13.block_sparse_moe.experts.1.w2.weight": "model-00016-of-00039.safetensors",
-    "layers.13.block_sparse_moe.experts.1.w3.weight": "model-00016-of-00039.safetensors",
-    "layers.13.block_sparse_moe.experts.2.w1.weight": "model-00016-of-00039.safetensors",
-    "layers.13.block_sparse_moe.experts.2.w2.weight": "model-00016-of-00039.safetensors",
-    "layers.13.block_sparse_moe.experts.2.w3.weight": "model-00017-of-00039.safetensors",
-    "layers.13.block_sparse_moe.experts.3.w1.weight": "model-00017-of-00039.safetensors",
-    "layers.13.block_sparse_moe.experts.3.w2.weight": "model-00017-of-00039.safetensors",
-    "layers.13.block_sparse_moe.experts.3.w3.weight": "model-00017-of-00039.safetensors",
-    "layers.13.block_sparse_moe.experts.4.w1.weight": "model-00017-of-00039.safetensors",
-    "layers.13.block_sparse_moe.experts.4.w2.weight": "model-00017-of-00039.safetensors",
-    "layers.13.block_sparse_moe.experts.4.w3.weight": "model-00017-of-00039.safetensors",
-    "layers.13.block_sparse_moe.experts.5.w1.weight": "model-00017-of-00039.safetensors",
-    "layers.13.block_sparse_moe.experts.5.w2.weight": "model-00017-of-00039.safetensors",
-    "layers.13.block_sparse_moe.experts.5.w3.weight": "model-00017-of-00039.safetensors",
-    "layers.13.block_sparse_moe.experts.6.w1.weight": "model-00017-of-00039.safetensors",
-    "layers.13.block_sparse_moe.experts.6.w2.weight": "model-00017-of-00039.safetensors",
-    "layers.13.block_sparse_moe.experts.6.w3.weight": "model-00017-of-00039.safetensors",
-    "layers.13.block_sparse_moe.experts.7.w1.weight": "model-00017-of-00039.safetensors",
-    "layers.13.block_sparse_moe.experts.7.w2.weight": "model-00017-of-00039.safetensors",
-    "layers.13.block_sparse_moe.experts.7.w3.weight": "model-00017-of-00039.safetensors",
-    "layers.13.block_sparse_moe.gate.weight": "model-00016-of-00039.safetensors",
-    "layers.13.input_layernorm.weight": "model-00017-of-00039.safetensors",
-    "layers.13.post_attention_layernorm.weight": "model-00017-of-00039.safetensors",
-    "layers.13.self_attn.k_proj.weight": "model-00016-of-00039.safetensors",
-    "layers.13.self_attn.o_proj.weight": "model-00016-of-00039.safetensors",
-    "layers.13.self_attn.q_proj.weight": "model-00016-of-00039.safetensors",
-    "layers.13.self_attn.v_proj.weight": "model-00016-of-00039.safetensors",
-    "layers.14.block_sparse_moe.experts.0.w1.weight": "model-00017-of-00039.safetensors",
-    "layers.14.block_sparse_moe.experts.0.w2.weight": "model-00017-of-00039.safetensors",
-    "layers.14.block_sparse_moe.experts.0.w3.weight": "model-00017-of-00039.safetensors",
-    "layers.14.block_sparse_moe.experts.1.w1.weight": "model-00017-of-00039.safetensors",
-    "layers.14.block_sparse_moe.experts.1.w2.weight": "model-00018-of-00039.safetensors",
-    "layers.14.block_sparse_moe.experts.1.w3.weight": "model-00018-of-00039.safetensors",
-    "layers.14.block_sparse_moe.experts.2.w1.weight": "model-00018-of-00039.safetensors",
-    "layers.14.block_sparse_moe.experts.2.w2.weight": "model-00018-of-00039.safetensors",
-    "layers.14.block_sparse_moe.experts.2.w3.weight": "model-00018-of-00039.safetensors",
-    "layers.14.block_sparse_moe.experts.3.w1.weight": "model-00018-of-00039.safetensors",
-    "layers.14.block_sparse_moe.experts.3.w2.weight": "model-00018-of-00039.safetensors",
-    "layers.14.block_sparse_moe.experts.3.w3.weight": "model-00018-of-00039.safetensors",
-    "layers.14.block_sparse_moe.experts.4.w1.weight": "model-00018-of-00039.safetensors",
-    "layers.14.block_sparse_moe.experts.4.w2.weight": "model-00018-of-00039.safetensors",
-    "layers.14.block_sparse_moe.experts.4.w3.weight": "model-00018-of-00039.safetensors",
-    "layers.14.block_sparse_moe.experts.5.w1.weight": "model-00018-of-00039.safetensors",
-    "layers.14.block_sparse_moe.experts.5.w2.weight": "model-00018-of-00039.safetensors",
-    "layers.14.block_sparse_moe.experts.5.w3.weight": "model-00018-of-00039.safetensors",
-    "layers.14.block_sparse_moe.experts.6.w1.weight": "model-00018-of-00039.safetensors",
-    "layers.14.block_sparse_moe.experts.6.w2.weight": "model-00018-of-00039.safetensors",
-    "layers.14.block_sparse_moe.experts.6.w3.weight": "model-00018-of-00039.safetensors",
-    "layers.14.block_sparse_moe.experts.7.w1.weight": "model-00018-of-00039.safetensors",
-    "layers.14.block_sparse_moe.experts.7.w2.weight": "model-00018-of-00039.safetensors",
-    "layers.14.block_sparse_moe.experts.7.w3.weight": "model-00018-of-00039.safetensors",
-    "layers.14.block_sparse_moe.gate.weight": "model-00017-of-00039.safetensors",
-    "layers.14.input_layernorm.weight": "model-00018-of-00039.safetensors",
-    "layers.14.post_attention_layernorm.weight": "model-00018-of-00039.safetensors",
-    "layers.14.self_attn.k_proj.weight": "model-00017-of-00039.safetensors",
-    "layers.14.self_attn.o_proj.weight": "model-00017-of-00039.safetensors",
-    "layers.14.self_attn.q_proj.weight": "model-00017-of-00039.safetensors",
-    "layers.14.self_attn.v_proj.weight": "model-00017-of-00039.safetensors",
-    "layers.15.block_sparse_moe.experts.0.w1.weight": "model-00019-of-00039.safetensors",
-    "layers.15.block_sparse_moe.experts.0.w2.weight": "model-00019-of-00039.safetensors",
-    "layers.15.block_sparse_moe.experts.0.w3.weight": "model-00019-of-00039.safetensors",
-    "layers.15.block_sparse_moe.experts.1.w1.weight": "model-00019-of-00039.safetensors",
-    "layers.15.block_sparse_moe.experts.1.w2.weight": "model-00019-of-00039.safetensors",
-    "layers.15.block_sparse_moe.experts.1.w3.weight": "model-00019-of-00039.safetensors",
-    "layers.15.block_sparse_moe.experts.2.w1.weight": "model-00019-of-00039.safetensors",
-    "layers.15.block_sparse_moe.experts.2.w2.weight": "model-00019-of-00039.safetensors",
-    "layers.15.block_sparse_moe.experts.2.w3.weight": "model-00019-of-00039.safetensors",
-    "layers.15.block_sparse_moe.experts.3.w1.weight": "model-00019-of-00039.safetensors",
-    "layers.15.block_sparse_moe.experts.3.w2.weight": "model-00019-of-00039.safetensors",
-    "layers.15.block_sparse_moe.experts.3.w3.weight": "model-00019-of-00039.safetensors",
-    "layers.15.block_sparse_moe.experts.4.w1.weight": "model-00019-of-00039.safetensors",
-    "layers.15.block_sparse_moe.experts.4.w2.weight": "model-00019-of-00039.safetensors",
-    "layers.15.block_sparse_moe.experts.4.w3.weight": "model-00019-of-00039.safetensors",
-    "layers.15.block_sparse_moe.experts.5.w1.weight": "model-00019-of-00039.safetensors",
-    "layers.15.block_sparse_moe.experts.5.w2.weight": "model-00019-of-00039.safetensors",
-    "layers.15.block_sparse_moe.experts.5.w3.weight": "model-00019-of-00039.safetensors",
-    "layers.15.block_sparse_moe.experts.6.w1.weight": "model-00019-of-00039.safetensors",
-    "layers.15.block_sparse_moe.experts.6.w2.weight": "model-00019-of-00039.safetensors",
-    "layers.15.block_sparse_moe.experts.6.w3.weight": "model-00019-of-00039.safetensors",
-    "layers.15.block_sparse_moe.experts.7.w1.weight": "model-00020-of-00039.safetensors",
-    "layers.15.block_sparse_moe.experts.7.w2.weight": "model-00020-of-00039.safetensors",
-    "layers.15.block_sparse_moe.experts.7.w3.weight": "model-00020-of-00039.safetensors",
-    "layers.15.block_sparse_moe.gate.weight": "model-00018-of-00039.safetensors",
-    "layers.15.input_layernorm.weight": "model-00020-of-00039.safetensors",
-    "layers.15.post_attention_layernorm.weight": "model-00020-of-00039.safetensors",
-    "layers.15.self_attn.k_proj.weight": "model-00018-of-00039.safetensors",
-    "layers.15.self_attn.o_proj.weight": "model-00018-of-00039.safetensors",
-    "layers.15.self_attn.q_proj.weight": "model-00018-of-00039.safetensors",
-    "layers.15.self_attn.v_proj.weight": "model-00018-of-00039.safetensors",
-    "layers.16.block_sparse_moe.experts.0.w1.weight": "model-00020-of-00039.safetensors",
-    "layers.16.block_sparse_moe.experts.0.w2.weight": "model-00020-of-00039.safetensors",
-    "layers.16.block_sparse_moe.experts.0.w3.weight": "model-00020-of-00039.safetensors",
-    "layers.16.block_sparse_moe.experts.1.w1.weight": "model-00020-of-00039.safetensors",
-    "layers.16.block_sparse_moe.experts.1.w2.weight": "model-00020-of-00039.safetensors",
-    "layers.16.block_sparse_moe.experts.1.w3.weight": "model-00020-of-00039.safetensors",
-    "layers.16.block_sparse_moe.experts.2.w1.weight": "model-00020-of-00039.safetensors",
-    "layers.16.block_sparse_moe.experts.2.w2.weight": "model-00020-of-00039.safetensors",
-    "layers.16.block_sparse_moe.experts.2.w3.weight": "model-00020-of-00039.safetensors",
-    "layers.16.block_sparse_moe.experts.3.w1.weight": "model-00020-of-00039.safetensors",
-    "layers.16.block_sparse_moe.experts.3.w2.weight": "model-00020-of-00039.safetensors",
-    "layers.16.block_sparse_moe.experts.3.w3.weight": "model-00020-of-00039.safetensors",
-    "layers.16.block_sparse_moe.experts.4.w1.weight": "model-00020-of-00039.safetensors",
-    "layers.16.block_sparse_moe.experts.4.w2.weight": "model-00020-of-00039.safetensors",
-    "layers.16.block_sparse_moe.experts.4.w3.weight": "model-00020-of-00039.safetensors",
-    "layers.16.block_sparse_moe.experts.5.w1.weight": "model-00020-of-00039.safetensors",
-    "layers.16.block_sparse_moe.experts.5.w2.weight": "model-00020-of-00039.safetensors",
-    "layers.16.block_sparse_moe.experts.5.w3.weight": "model-00021-of-00039.safetensors",
-    "layers.16.block_sparse_moe.experts.6.w1.weight": "model-00021-of-00039.safetensors",
-    "layers.16.block_sparse_moe.experts.6.w2.weight": "model-00021-of-00039.safetensors",
-    "layers.16.block_sparse_moe.experts.6.w3.weight": "model-00021-of-00039.safetensors",
-    "layers.16.block_sparse_moe.experts.7.w1.weight": "model-00021-of-00039.safetensors",
-    "layers.16.block_sparse_moe.experts.7.w2.weight": "model-00021-of-00039.safetensors",
-    "layers.16.block_sparse_moe.experts.7.w3.weight": "model-00021-of-00039.safetensors",
-    "layers.16.block_sparse_moe.gate.weight": "model-00020-of-00039.safetensors",
-    "layers.16.input_layernorm.weight": "model-00021-of-00039.safetensors",
-    "layers.16.post_attention_layernorm.weight": "model-00021-of-00039.safetensors",
-    "layers.16.self_attn.k_proj.weight": "model-00020-of-00039.safetensors",
-    "layers.16.self_attn.o_proj.weight": "model-00020-of-00039.safetensors",
-    "layers.16.self_attn.q_proj.weight": "model-00020-of-00039.safetensors",
-    "layers.16.self_attn.v_proj.weight": "model-00020-of-00039.safetensors",
-    "layers.17.block_sparse_moe.experts.0.w1.weight": "model-00021-of-00039.safetensors",
-    "layers.17.block_sparse_moe.experts.0.w2.weight": "model-00021-of-00039.safetensors",
-    "layers.17.block_sparse_moe.experts.0.w3.weight": "model-00021-of-00039.safetensors",
-    "layers.17.block_sparse_moe.experts.1.w1.weight": "model-00021-of-00039.safetensors",
-    "layers.17.block_sparse_moe.experts.1.w2.weight": "model-00021-of-00039.safetensors",
-    "layers.17.block_sparse_moe.experts.1.w3.weight": "model-00021-of-00039.safetensors",
-    "layers.17.block_sparse_moe.experts.2.w1.weight": "model-00021-of-00039.safetensors",
-    "layers.17.block_sparse_moe.experts.2.w2.weight": "model-00021-of-00039.safetensors",
-    "layers.17.block_sparse_moe.experts.2.w3.weight": "model-00021-of-00039.safetensors",
-    "layers.17.block_sparse_moe.experts.3.w1.weight": "model-00021-of-00039.safetensors",
-    "layers.17.block_sparse_moe.experts.3.w2.weight": "model-00021-of-00039.safetensors",
-    "layers.17.block_sparse_moe.experts.3.w3.weight": "model-00021-of-00039.safetensors",
-    "layers.17.block_sparse_moe.experts.4.w1.weight": "model-00021-of-00039.safetensors",
-    "layers.17.block_sparse_moe.experts.4.w2.weight": "model-00022-of-00039.safetensors",
-    "layers.17.block_sparse_moe.experts.4.w3.weight": "model-00022-of-00039.safetensors",
-    "layers.17.block_sparse_moe.experts.5.w1.weight": "model-00022-of-00039.safetensors",
-    "layers.17.block_sparse_moe.experts.5.w2.weight": "model-00022-of-00039.safetensors",
-    "layers.17.block_sparse_moe.experts.5.w3.weight": "model-00022-of-00039.safetensors",
-    "layers.17.block_sparse_moe.experts.6.w1.weight": "model-00022-of-00039.safetensors",
-    "layers.17.block_sparse_moe.experts.6.w2.weight": "model-00022-of-00039.safetensors",
-    "layers.17.block_sparse_moe.experts.6.w3.weight": "model-00022-of-00039.safetensors",
-    "layers.17.block_sparse_moe.experts.7.w1.weight": "model-00022-of-00039.safetensors",
-    "layers.17.block_sparse_moe.experts.7.w2.weight": "model-00022-of-00039.safetensors",
-    "layers.17.block_sparse_moe.experts.7.w3.weight": "model-00022-of-00039.safetensors",
-    "layers.17.block_sparse_moe.gate.weight": "model-00021-of-00039.safetensors",
-    "layers.17.input_layernorm.weight": "model-00022-of-00039.safetensors",
-    "layers.17.post_attention_layernorm.weight": "model-00022-of-00039.safetensors",
-    "layers.17.self_attn.k_proj.weight": "model-00021-of-00039.safetensors",
-    "layers.17.self_attn.o_proj.weight": "model-00021-of-00039.safetensors",
-    "layers.17.self_attn.q_proj.weight": "model-00021-of-00039.safetensors",
-    "layers.17.self_attn.v_proj.weight": "model-00021-of-00039.safetensors",
-    "layers.18.block_sparse_moe.experts.0.w1.weight": "model-00022-of-00039.safetensors",
-    "layers.18.block_sparse_moe.experts.0.w2.weight": "model-00022-of-00039.safetensors",
-    "layers.18.block_sparse_moe.experts.0.w3.weight": "model-00022-of-00039.safetensors",
-    "layers.18.block_sparse_moe.experts.1.w1.weight": "model-00022-of-00039.safetensors",
-    "layers.18.block_sparse_moe.experts.1.w2.weight": "model-00022-of-00039.safetensors",
-    "layers.18.block_sparse_moe.experts.1.w3.weight": "model-00022-of-00039.safetensors",
-    "layers.18.block_sparse_moe.experts.2.w1.weight": "model-00022-of-00039.safetensors",
-    "layers.18.block_sparse_moe.experts.2.w2.weight": "model-00022-of-00039.safetensors",
-    "layers.18.block_sparse_moe.experts.2.w3.weight": "model-00022-of-00039.safetensors",
-    "layers.18.block_sparse_moe.experts.3.w1.weight": "model-00023-of-00039.safetensors",
-    "layers.18.block_sparse_moe.experts.3.w2.weight": "model-00023-of-00039.safetensors",
-    "layers.18.block_sparse_moe.experts.3.w3.weight": "model-00023-of-00039.safetensors",
-    "layers.18.block_sparse_moe.experts.4.w1.weight": "model-00023-of-00039.safetensors",
-    "layers.18.block_sparse_moe.experts.4.w2.weight": "model-00023-of-00039.safetensors",
-    "layers.18.block_sparse_moe.experts.4.w3.weight": "model-00023-of-00039.safetensors",
-    "layers.18.block_sparse_moe.experts.5.w1.weight": "model-00023-of-00039.safetensors",
-    "layers.18.block_sparse_moe.experts.5.w2.weight": "model-00023-of-00039.safetensors",
-    "layers.18.block_sparse_moe.experts.5.w3.weight": "model-00023-of-00039.safetensors",
-    "layers.18.block_sparse_moe.experts.6.w1.weight": "model-00023-of-00039.safetensors",
-    "layers.18.block_sparse_moe.experts.6.w2.weight": "model-00023-of-00039.safetensors",
-    "layers.18.block_sparse_moe.experts.6.w3.weight": "model-00023-of-00039.safetensors",
-    "layers.18.block_sparse_moe.experts.7.w1.weight": "model-00023-of-00039.safetensors",
-    "layers.18.block_sparse_moe.experts.7.w2.weight": "model-00023-of-00039.safetensors",
-    "layers.18.block_sparse_moe.experts.7.w3.weight": "model-00023-of-00039.safetensors",
-    "layers.18.block_sparse_moe.gate.weight": "model-00022-of-00039.safetensors",
-    "layers.18.input_layernorm.weight": "model-00023-of-00039.safetensors",
-    "layers.18.post_attention_layernorm.weight": "model-00023-of-00039.safetensors",
-    "layers.18.self_attn.k_proj.weight": "model-00022-of-00039.safetensors",
-    "layers.18.self_attn.o_proj.weight": "model-00022-of-00039.safetensors",
-    "layers.18.self_attn.q_proj.weight": "model-00022-of-00039.safetensors",
-    "layers.18.self_attn.v_proj.weight": "model-00022-of-00039.safetensors",
-    "layers.19.block_sparse_moe.experts.0.w1.weight": "model-00023-of-00039.safetensors",
-    "layers.19.block_sparse_moe.experts.0.w2.weight": "model-00023-of-00039.safetensors",
-    "layers.19.block_sparse_moe.experts.0.w3.weight": "model-00023-of-00039.safetensors",
-    "layers.19.block_sparse_moe.experts.1.w1.weight": "model-00023-of-00039.safetensors",
-    "layers.19.block_sparse_moe.experts.1.w2.weight": "model-00023-of-00039.safetensors",
-    "layers.19.block_sparse_moe.experts.1.w3.weight": "model-00024-of-00039.safetensors",
-    "layers.19.block_sparse_moe.experts.2.w1.weight": "model-00024-of-00039.safetensors",
-    "layers.19.block_sparse_moe.experts.2.w2.weight": "model-00024-of-00039.safetensors",
-    "layers.19.block_sparse_moe.experts.2.w3.weight": "model-00024-of-00039.safetensors",
-    "layers.19.block_sparse_moe.experts.3.w1.weight": "model-00024-of-00039.safetensors",
-    "layers.19.block_sparse_moe.experts.3.w2.weight": "model-00024-of-00039.safetensors",
-    "layers.19.block_sparse_moe.experts.3.w3.weight": "model-00024-of-00039.safetensors",
-    "layers.19.block_sparse_moe.experts.4.w1.weight": "model-00024-of-00039.safetensors",
-    "layers.19.block_sparse_moe.experts.4.w2.weight": "model-00024-of-00039.safetensors",
-    "layers.19.block_sparse_moe.experts.4.w3.weight": "model-00024-of-00039.safetensors",
-    "layers.19.block_sparse_moe.experts.5.w1.weight": "model-00024-of-00039.safetensors",
-    "layers.19.block_sparse_moe.experts.5.w2.weight": "model-00024-of-00039.safetensors",
-    "layers.19.block_sparse_moe.experts.5.w3.weight": "model-00024-of-00039.safetensors",
-    "layers.19.block_sparse_moe.experts.6.w1.weight": "model-00024-of-00039.safetensors",
-    "layers.19.block_sparse_moe.experts.6.w2.weight": "model-00024-of-00039.safetensors",
-    "layers.19.block_sparse_moe.experts.6.w3.weight": "model-00024-of-00039.safetensors",
-    "layers.19.block_sparse_moe.experts.7.w1.weight": "model-00024-of-00039.safetensors",
-    "layers.19.block_sparse_moe.experts.7.w2.weight": "model-00024-of-00039.safetensors",
-    "layers.19.block_sparse_moe.experts.7.w3.weight": "model-00024-of-00039.safetensors",
-    "layers.19.block_sparse_moe.gate.weight": "model-00023-of-00039.safetensors",
-    "layers.19.input_layernorm.weight": "model-00024-of-00039.safetensors",
-    "layers.19.post_attention_layernorm.weight": "model-00024-of-00039.safetensors",
-    "layers.19.self_attn.k_proj.weight": "model-00023-of-00039.safetensors",
-    "layers.19.self_attn.o_proj.weight": "model-00023-of-00039.safetensors",
-    "layers.19.self_attn.q_proj.weight": "model-00023-of-00039.safetensors",
-    "layers.19.self_attn.v_proj.weight": "model-00023-of-00039.safetensors",
-    "layers.2.block_sparse_moe.experts.0.w1.weight": "model-00003-of-00039.safetensors",
-    "layers.2.block_sparse_moe.experts.0.w2.weight": "model-00003-of-00039.safetensors",
-    "layers.2.block_sparse_moe.experts.0.w3.weight": "model-00003-of-00039.safetensors",
-    "layers.2.block_sparse_moe.experts.1.w1.weight": "model-00003-of-00039.safetensors",
-    "layers.2.block_sparse_moe.experts.1.w2.weight": "model-00003-of-00039.safetensors",
-    "layers.2.block_sparse_moe.experts.1.w3.weight": "model-00003-of-00039.safetensors",
-    "layers.2.block_sparse_moe.experts.2.w1.weight": "model-00003-of-00039.safetensors",
-    "layers.2.block_sparse_moe.experts.2.w2.weight": "model-00003-of-00039.safetensors",
-    "layers.2.block_sparse_moe.experts.2.w3.weight": "model-00003-of-00039.safetensors",
-    "layers.2.block_sparse_moe.experts.3.w1.weight": "model-00003-of-00039.safetensors",
-    "layers.2.block_sparse_moe.experts.3.w2.weight": "model-00004-of-00039.safetensors",
-    "layers.2.block_sparse_moe.experts.3.w3.weight": "model-00004-of-00039.safetensors",
-    "layers.2.block_sparse_moe.experts.4.w1.weight": "model-00004-of-00039.safetensors",
-    "layers.2.block_sparse_moe.experts.4.w2.weight": "model-00004-of-00039.safetensors",
-    "layers.2.block_sparse_moe.experts.4.w3.weight": "model-00004-of-00039.safetensors",
-    "layers.2.block_sparse_moe.experts.5.w1.weight": "model-00004-of-00039.safetensors",
-    "layers.2.block_sparse_moe.experts.5.w2.weight": "model-00004-of-00039.safetensors",
-    "layers.2.block_sparse_moe.experts.5.w3.weight": "model-00004-of-00039.safetensors",
-    "layers.2.block_sparse_moe.experts.6.w1.weight": "model-00004-of-00039.safetensors",
-    "layers.2.block_sparse_moe.experts.6.w2.weight": "model-00004-of-00039.safetensors",
-    "layers.2.block_sparse_moe.experts.6.w3.weight": "model-00004-of-00039.safetensors",
-    "layers.2.block_sparse_moe.experts.7.w1.weight": "model-00004-of-00039.safetensors",
-    "layers.2.block_sparse_moe.experts.7.w2.weight": "model-00004-of-00039.safetensors",
-    "layers.2.block_sparse_moe.experts.7.w3.weight": "model-00004-of-00039.safetensors",
-    "layers.2.block_sparse_moe.gate.weight": "model-00003-of-00039.safetensors",
-    "layers.2.input_layernorm.weight": "model-00004-of-00039.safetensors",
-    "layers.2.post_attention_layernorm.weight": "model-00004-of-00039.safetensors",
-    "layers.2.self_attn.k_proj.weight": "model-00003-of-00039.safetensors",
-    "layers.2.self_attn.o_proj.weight": "model-00003-of-00039.safetensors",
-    "layers.2.self_attn.q_proj.weight": "model-00003-of-00039.safetensors",
-    "layers.2.self_attn.v_proj.weight": "model-00003-of-00039.safetensors",
-    "layers.20.block_sparse_moe.experts.0.w1.weight": "model-00024-of-00039.safetensors",
-    "layers.20.block_sparse_moe.experts.0.w2.weight": "model-00025-of-00039.safetensors",
-    "layers.20.block_sparse_moe.experts.0.w3.weight": "model-00025-of-00039.safetensors",
-    "layers.20.block_sparse_moe.experts.1.w1.weight": "model-00025-of-00039.safetensors",
-    "layers.20.block_sparse_moe.experts.1.w2.weight": "model-00025-of-00039.safetensors",
-    "layers.20.block_sparse_moe.experts.1.w3.weight": "model-00025-of-00039.safetensors",
-    "layers.20.block_sparse_moe.experts.2.w1.weight": "model-00025-of-00039.safetensors",
-    "layers.20.block_sparse_moe.experts.2.w2.weight": "model-00025-of-00039.safetensors",
-    "layers.20.block_sparse_moe.experts.2.w3.weight": "model-00025-of-00039.safetensors",
-    "layers.20.block_sparse_moe.experts.3.w1.weight": "model-00025-of-00039.safetensors",
-    "layers.20.block_sparse_moe.experts.3.w2.weight": "model-00025-of-00039.safetensors",
-    "layers.20.block_sparse_moe.experts.3.w3.weight": "model-00025-of-00039.safetensors",
-    "layers.20.block_sparse_moe.experts.4.w1.weight": "model-00025-of-00039.safetensors",
-    "layers.20.block_sparse_moe.experts.4.w2.weight": "model-00025-of-00039.safetensors",
-    "layers.20.block_sparse_moe.experts.4.w3.weight": "model-00025-of-00039.safetensors",
-    "layers.20.block_sparse_moe.experts.5.w1.weight": "model-00025-of-00039.safetensors",
-    "layers.20.block_sparse_moe.experts.5.w2.weight": "model-00025-of-00039.safetensors",
-    "layers.20.block_sparse_moe.experts.5.w3.weight": "model-00025-of-00039.safetensors",
-    "layers.20.block_sparse_moe.experts.6.w1.weight": "model-00025-of-00039.safetensors",
-    "layers.20.block_sparse_moe.experts.6.w2.weight": "model-00025-of-00039.safetensors",
-    "layers.20.block_sparse_moe.experts.6.w3.weight": "model-00025-of-00039.safetensors",
-    "layers.20.block_sparse_moe.experts.7.w1.weight": "model-00025-of-00039.safetensors",
-    "layers.20.block_sparse_moe.experts.7.w2.weight": "model-00026-of-00039.safetensors",
-    "layers.20.block_sparse_moe.experts.7.w3.weight": "model-00026-of-00039.safetensors",
-    "layers.20.block_sparse_moe.gate.weight": "model-00024-of-00039.safetensors",
-    "layers.20.input_layernorm.weight": "model-00026-of-00039.safetensors",
-    "layers.20.post_attention_layernorm.weight": "model-00026-of-00039.safetensors",
-    "layers.20.self_attn.k_proj.weight": "model-00024-of-00039.safetensors",
-    "layers.20.self_attn.o_proj.weight": "model-00024-of-00039.safetensors",
-    "layers.20.self_attn.q_proj.weight": "model-00024-of-00039.safetensors",
-    "layers.20.self_attn.v_proj.weight": "model-00024-of-00039.safetensors",
-    "layers.21.block_sparse_moe.experts.0.w1.weight": "model-00026-of-00039.safetensors",
-    "layers.21.block_sparse_moe.experts.0.w2.weight": "model-00026-of-00039.safetensors",
-    "layers.21.block_sparse_moe.experts.0.w3.weight": "model-00026-of-00039.safetensors",
-    "layers.21.block_sparse_moe.experts.1.w1.weight": "model-00026-of-00039.safetensors",
-    "layers.21.block_sparse_moe.experts.1.w2.weight": "model-00026-of-00039.safetensors",
-    "layers.21.block_sparse_moe.experts.1.w3.weight": "model-00026-of-00039.safetensors",
-    "layers.21.block_sparse_moe.experts.2.w1.weight": "model-00026-of-00039.safetensors",
-    "layers.21.block_sparse_moe.experts.2.w2.weight": "model-00026-of-00039.safetensors",
-    "layers.21.block_sparse_moe.experts.2.w3.weight": "model-00026-of-00039.safetensors",
-    "layers.21.block_sparse_moe.experts.3.w1.weight": "model-00026-of-00039.safetensors",
-    "layers.21.block_sparse_moe.experts.3.w2.weight": "model-00026-of-00039.safetensors",
-    "layers.21.block_sparse_moe.experts.3.w3.weight": "model-00026-of-00039.safetensors",
-    "layers.21.block_sparse_moe.experts.4.w1.weight": "model-00026-of-00039.safetensors",
-    "layers.21.block_sparse_moe.experts.4.w2.weight": "model-00026-of-00039.safetensors",
-    "layers.21.block_sparse_moe.experts.4.w3.weight": "model-00026-of-00039.safetensors",
-    "layers.21.block_sparse_moe.experts.5.w1.weight": "model-00026-of-00039.safetensors",
-    "layers.21.block_sparse_moe.experts.5.w2.weight": "model-00026-of-00039.safetensors",
-    "layers.21.block_sparse_moe.experts.5.w3.weight": "model-00026-of-00039.safetensors",
-    "layers.21.block_sparse_moe.experts.6.w1.weight": "model-00027-of-00039.safetensors",
-    "layers.21.block_sparse_moe.experts.6.w2.weight": "model-00027-of-00039.safetensors",
-    "layers.21.block_sparse_moe.experts.6.w3.weight": "model-00027-of-00039.safetensors",
-    "layers.21.block_sparse_moe.experts.7.w1.weight": "model-00027-of-00039.safetensors",
-    "layers.21.block_sparse_moe.experts.7.w2.weight": "model-00027-of-00039.safetensors",
-    "layers.21.block_sparse_moe.experts.7.w3.weight": "model-00027-of-00039.safetensors",
-    "layers.21.block_sparse_moe.gate.weight": "model-00026-of-00039.safetensors",
-    "layers.21.input_layernorm.weight": "model-00027-of-00039.safetensors",
-    "layers.21.post_attention_layernorm.weight": "model-00027-of-00039.safetensors",
-    "layers.21.self_attn.k_proj.weight": "model-00026-of-00039.safetensors",
-    "layers.21.self_attn.o_proj.weight": "model-00026-of-00039.safetensors",
-    "layers.21.self_attn.q_proj.weight": "model-00026-of-00039.safetensors",
-    "layers.21.self_attn.v_proj.weight": "model-00026-of-00039.safetensors",
-    "layers.22.block_sparse_moe.experts.0.w1.weight": "model-00027-of-00039.safetensors",
-    "layers.22.block_sparse_moe.experts.0.w2.weight": "model-00027-of-00039.safetensors",
-    "layers.22.block_sparse_moe.experts.0.w3.weight": "model-00027-of-00039.safetensors",
-    "layers.22.block_sparse_moe.experts.1.w1.weight": "model-00027-of-00039.safetensors",
-    "layers.22.block_sparse_moe.experts.1.w2.weight": "model-00027-of-00039.safetensors",
-    "layers.22.block_sparse_moe.experts.1.w3.weight": "model-00027-of-00039.safetensors",
-    "layers.22.block_sparse_moe.experts.2.w1.weight": "model-00027-of-00039.safetensors",
-    "layers.22.block_sparse_moe.experts.2.w2.weight": "model-00027-of-00039.safetensors",
-    "layers.22.block_sparse_moe.experts.2.w3.weight": "model-00027-of-00039.safetensors",
-    "layers.22.block_sparse_moe.experts.3.w1.weight": "model-00027-of-00039.safetensors",
-    "layers.22.block_sparse_moe.experts.3.w2.weight": "model-00027-of-00039.safetensors",
-    "layers.22.block_sparse_moe.experts.3.w3.weight": "model-00027-of-00039.safetensors",
-    "layers.22.block_sparse_moe.experts.4.w1.weight": "model-00027-of-00039.safetensors",
-    "layers.22.block_sparse_moe.experts.4.w2.weight": "model-00027-of-00039.safetensors",
-    "layers.22.block_sparse_moe.experts.4.w3.weight": "model-00028-of-00039.safetensors",
-    "layers.22.block_sparse_moe.experts.5.w1.weight": "model-00028-of-00039.safetensors",
-    "layers.22.block_sparse_moe.experts.5.w2.weight": "model-00028-of-00039.safetensors",
-    "layers.22.block_sparse_moe.experts.5.w3.weight": "model-00028-of-00039.safetensors",
-    "layers.22.block_sparse_moe.experts.6.w1.weight": "model-00028-of-00039.safetensors",
-    "layers.22.block_sparse_moe.experts.6.w2.weight": "model-00028-of-00039.safetensors",
-    "layers.22.block_sparse_moe.experts.6.w3.weight": "model-00028-of-00039.safetensors",
-    "layers.22.block_sparse_moe.experts.7.w1.weight": "model-00028-of-00039.safetensors",
-    "layers.22.block_sparse_moe.experts.7.w2.weight": "model-00028-of-00039.safetensors",
-    "layers.22.block_sparse_moe.experts.7.w3.weight": "model-00028-of-00039.safetensors",
-    "layers.22.block_sparse_moe.gate.weight": "model-00027-of-00039.safetensors",
-    "layers.22.input_layernorm.weight": "model-00028-of-00039.safetensors",
-    "layers.22.post_attention_layernorm.weight": "model-00028-of-00039.safetensors",
-    "layers.22.self_attn.k_proj.weight": "model-00027-of-00039.safetensors",
-    "layers.22.self_attn.o_proj.weight": "model-00027-of-00039.safetensors",
-    "layers.22.self_attn.q_proj.weight": "model-00027-of-00039.safetensors",
-    "layers.22.self_attn.v_proj.weight": "model-00027-of-00039.safetensors",
-    "layers.23.block_sparse_moe.experts.0.w1.weight": "model-00028-of-00039.safetensors",
-    "layers.23.block_sparse_moe.experts.0.w2.weight": "model-00028-of-00039.safetensors",
-    "layers.23.block_sparse_moe.experts.0.w3.weight": "model-00028-of-00039.safetensors",
-    "layers.23.block_sparse_moe.experts.1.w1.weight": "model-00028-of-00039.safetensors",
-    "layers.23.block_sparse_moe.experts.1.w2.weight": "model-00028-of-00039.safetensors",
-    "layers.23.block_sparse_moe.experts.1.w3.weight": "model-00028-of-00039.safetensors",
-    "layers.23.block_sparse_moe.experts.2.w1.weight": "model-00028-of-00039.safetensors",
-    "layers.23.block_sparse_moe.experts.2.w2.weight": "model-00028-of-00039.safetensors",
-    "layers.23.block_sparse_moe.experts.2.w3.weight": "model-00028-of-00039.safetensors",
-    "layers.23.block_sparse_moe.experts.3.w1.weight": "model-00028-of-00039.safetensors",
-    "layers.23.block_sparse_moe.experts.3.w2.weight": "model-00029-of-00039.safetensors",
-    "layers.23.block_sparse_moe.experts.3.w3.weight": "model-00029-of-00039.safetensors",
-    "layers.23.block_sparse_moe.experts.4.w1.weight": "model-00029-of-00039.safetensors",
-    "layers.23.block_sparse_moe.experts.4.w2.weight": "model-00029-of-00039.safetensors",
-    "layers.23.block_sparse_moe.experts.4.w3.weight": "model-00029-of-00039.safetensors",
-    "layers.23.block_sparse_moe.experts.5.w1.weight": "model-00029-of-00039.safetensors",
-    "layers.23.block_sparse_moe.experts.5.w2.weight": "model-00029-of-00039.safetensors",
-    "layers.23.block_sparse_moe.experts.5.w3.weight": "model-00029-of-00039.safetensors",
-    "layers.23.block_sparse_moe.experts.6.w1.weight": "model-00029-of-00039.safetensors",
-    "layers.23.block_sparse_moe.experts.6.w2.weight": "model-00029-of-00039.safetensors",
-    "layers.23.block_sparse_moe.experts.6.w3.weight": "model-00029-of-00039.safetensors",
-    "layers.23.block_sparse_moe.experts.7.w1.weight": "model-00029-of-00039.safetensors",
-    "layers.23.block_sparse_moe.experts.7.w2.weight": "model-00029-of-00039.safetensors",
-    "layers.23.block_sparse_moe.experts.7.w3.weight": "model-00029-of-00039.safetensors",
-    "layers.23.block_sparse_moe.gate.weight": "model-00028-of-00039.safetensors",
-    "layers.23.input_layernorm.weight": "model-00029-of-00039.safetensors",
-    "layers.23.post_attention_layernorm.weight": "model-00029-of-00039.safetensors",
-    "layers.23.self_attn.k_proj.weight": "model-00028-of-00039.safetensors",
-    "layers.23.self_attn.o_proj.weight": "model-00028-of-00039.safetensors",
-    "layers.23.self_attn.q_proj.weight": "model-00028-of-00039.safetensors",
-    "layers.23.self_attn.v_proj.weight": "model-00028-of-00039.safetensors",
-    "layers.24.block_sparse_moe.experts.0.w1.weight": "model-00029-of-00039.safetensors",
-    "layers.24.block_sparse_moe.experts.0.w2.weight": "model-00029-of-00039.safetensors",
-    "layers.24.block_sparse_moe.experts.0.w3.weight": "model-00029-of-00039.safetensors",
-    "layers.24.block_sparse_moe.experts.1.w1.weight": "model-00029-of-00039.safetensors",
-    "layers.24.block_sparse_moe.experts.1.w2.weight": "model-00029-of-00039.safetensors",
-    "layers.24.block_sparse_moe.experts.1.w3.weight": "model-00029-of-00039.safetensors",
-    "layers.24.block_sparse_moe.experts.2.w1.weight": "model-00030-of-00039.safetensors",
-    "layers.24.block_sparse_moe.experts.2.w2.weight": "model-00030-of-00039.safetensors",
-    "layers.24.block_sparse_moe.experts.2.w3.weight": "model-00030-of-00039.safetensors",
-    "layers.24.block_sparse_moe.experts.3.w1.weight": "model-00030-of-00039.safetensors",
-    "layers.24.block_sparse_moe.experts.3.w2.weight": "model-00030-of-00039.safetensors",
-    "layers.24.block_sparse_moe.experts.3.w3.weight": "model-00030-of-00039.safetensors",
-    "layers.24.block_sparse_moe.experts.4.w1.weight": "model-00030-of-00039.safetensors",
-    "layers.24.block_sparse_moe.experts.4.w2.weight": "model-00030-of-00039.safetensors",
-    "layers.24.block_sparse_moe.experts.4.w3.weight": "model-00030-of-00039.safetensors",
-    "layers.24.block_sparse_moe.experts.5.w1.weight": "model-00030-of-00039.safetensors",
-    "layers.24.block_sparse_moe.experts.5.w2.weight": "model-00030-of-00039.safetensors",
-    "layers.24.block_sparse_moe.experts.5.w3.weight": "model-00030-of-00039.safetensors",
-    "layers.24.block_sparse_moe.experts.6.w1.weight": "model-00030-of-00039.safetensors",
-    "layers.24.block_sparse_moe.experts.6.w2.weight": "model-00030-of-00039.safetensors",
-    "layers.24.block_sparse_moe.experts.6.w3.weight": "model-00030-of-00039.safetensors",
-    "layers.24.block_sparse_moe.experts.7.w1.weight": "model-00030-of-00039.safetensors",
-    "layers.24.block_sparse_moe.experts.7.w2.weight": "model-00030-of-00039.safetensors",
-    "layers.24.block_sparse_moe.experts.7.w3.weight": "model-00030-of-00039.safetensors",
-    "layers.24.block_sparse_moe.gate.weight": "model-00029-of-00039.safetensors",
-    "layers.24.input_layernorm.weight": "model-00030-of-00039.safetensors",
-    "layers.24.post_attention_layernorm.weight": "model-00030-of-00039.safetensors",
-    "layers.24.self_attn.k_proj.weight": "model-00029-of-00039.safetensors",
-    "layers.24.self_attn.o_proj.weight": "model-00029-of-00039.safetensors",
-    "layers.24.self_attn.q_proj.weight": "model-00029-of-00039.safetensors",
-    "layers.24.self_attn.v_proj.weight": "model-00029-of-00039.safetensors",
-    "layers.25.block_sparse_moe.experts.0.w1.weight": "model-00030-of-00039.safetensors",
-    "layers.25.block_sparse_moe.experts.0.w2.weight": "model-00030-of-00039.safetensors",
-    "layers.25.block_sparse_moe.experts.0.w3.weight": "model-00031-of-00039.safetensors",
-    "layers.25.block_sparse_moe.experts.1.w1.weight": "model-00031-of-00039.safetensors",
-    "layers.25.block_sparse_moe.experts.1.w2.weight": "model-00031-of-00039.safetensors",
-    "layers.25.block_sparse_moe.experts.1.w3.weight": "model-00031-of-00039.safetensors",
-    "layers.25.block_sparse_moe.experts.2.w1.weight": "model-00031-of-00039.safetensors",
-    "layers.25.block_sparse_moe.experts.2.w2.weight": "model-00031-of-00039.safetensors",
-    "layers.25.block_sparse_moe.experts.2.w3.weight": "model-00031-of-00039.safetensors",
-    "layers.25.block_sparse_moe.experts.3.w1.weight": "model-00031-of-00039.safetensors",
-    "layers.25.block_sparse_moe.experts.3.w2.weight": "model-00031-of-00039.safetensors",
-    "layers.25.block_sparse_moe.experts.3.w3.weight": "model-00031-of-00039.safetensors",
-    "layers.25.block_sparse_moe.experts.4.w1.weight": "model-00031-of-00039.safetensors",
-    "layers.25.block_sparse_moe.experts.4.w2.weight": "model-00031-of-00039.safetensors",
-    "layers.25.block_sparse_moe.experts.4.w3.weight": "model-00031-of-00039.safetensors",
-    "layers.25.block_sparse_moe.experts.5.w1.weight": "model-00031-of-00039.safetensors",
-    "layers.25.block_sparse_moe.experts.5.w2.weight": "model-00031-of-00039.safetensors",
-    "layers.25.block_sparse_moe.experts.5.w3.weight": "model-00031-of-00039.safetensors",
-    "layers.25.block_sparse_moe.experts.6.w1.weight": "model-00031-of-00039.safetensors",
-    "layers.25.block_sparse_moe.experts.6.w2.weight": "model-00031-of-00039.safetensors",
-    "layers.25.block_sparse_moe.experts.6.w3.weight": "model-00031-of-00039.safetensors",
-    "layers.25.block_sparse_moe.experts.7.w1.weight": "model-00031-of-00039.safetensors",
-    "layers.25.block_sparse_moe.experts.7.w2.weight": "model-00031-of-00039.safetensors",
-    "layers.25.block_sparse_moe.experts.7.w3.weight": "model-00032-of-00039.safetensors",
-    "layers.25.block_sparse_moe.gate.weight": "model-00030-of-00039.safetensors",
-    "layers.25.input_layernorm.weight": "model-00032-of-00039.safetensors",
-    "layers.25.post_attention_layernorm.weight": "model-00032-of-00039.safetensors",
-    "layers.25.self_attn.k_proj.weight": "model-00030-of-00039.safetensors",
-    "layers.25.self_attn.o_proj.weight": "model-00030-of-00039.safetensors",
-    "layers.25.self_attn.q_proj.weight": "model-00030-of-00039.safetensors",
-    "layers.25.self_attn.v_proj.weight": "model-00030-of-00039.safetensors",
-    "layers.26.block_sparse_moe.experts.0.w1.weight": "model-00032-of-00039.safetensors",
-    "layers.26.block_sparse_moe.experts.0.w2.weight": "model-00032-of-00039.safetensors",
-    "layers.26.block_sparse_moe.experts.0.w3.weight": "model-00032-of-00039.safetensors",
-    "layers.26.block_sparse_moe.experts.1.w1.weight": "model-00032-of-00039.safetensors",
-    "layers.26.block_sparse_moe.experts.1.w2.weight": "model-00032-of-00039.safetensors",
-    "layers.26.block_sparse_moe.experts.1.w3.weight": "model-00032-of-00039.safetensors",
-    "layers.26.block_sparse_moe.experts.2.w1.weight": "model-00032-of-00039.safetensors",
-    "layers.26.block_sparse_moe.experts.2.w2.weight": "model-00032-of-00039.safetensors",
-    "layers.26.block_sparse_moe.experts.2.w3.weight": "model-00032-of-00039.safetensors",
-    "layers.26.block_sparse_moe.experts.3.w1.weight": "model-00032-of-00039.safetensors",
-    "layers.26.block_sparse_moe.experts.3.w2.weight": "model-00032-of-00039.safetensors",
-    "layers.26.block_sparse_moe.experts.3.w3.weight": "model-00032-of-00039.safetensors",
-    "layers.26.block_sparse_moe.experts.4.w1.weight": "model-00032-of-00039.safetensors",
-    "layers.26.block_sparse_moe.experts.4.w2.weight": "model-00032-of-00039.safetensors",
-    "layers.26.block_sparse_moe.experts.4.w3.weight": "model-00032-of-00039.safetensors",
-    "layers.26.block_sparse_moe.experts.5.w1.weight": "model-00032-of-00039.safetensors",
-    "layers.26.block_sparse_moe.experts.5.w2.weight": "model-00032-of-00039.safetensors",
-    "layers.26.block_sparse_moe.experts.5.w3.weight": "model-00032-of-00039.safetensors",
-    "layers.26.block_sparse_moe.experts.6.w1.weight": "model-00032-of-00039.safetensors",
-    "layers.26.block_sparse_moe.experts.6.w2.weight": "model-00033-of-00039.safetensors",
-    "layers.26.block_sparse_moe.experts.6.w3.weight": "model-00033-of-00039.safetensors",
-    "layers.26.block_sparse_moe.experts.7.w1.weight": "model-00033-of-00039.safetensors",
-    "layers.26.block_sparse_moe.experts.7.w2.weight": "model-00033-of-00039.safetensors",
-    "layers.26.block_sparse_moe.experts.7.w3.weight": "model-00033-of-00039.safetensors",
-    "layers.26.block_sparse_moe.gate.weight": "model-00032-of-00039.safetensors",
-    "layers.26.input_layernorm.weight": "model-00033-of-00039.safetensors",
-    "layers.26.post_attention_layernorm.weight": "model-00033-of-00039.safetensors",
-    "layers.26.self_attn.k_proj.weight": "model-00032-of-00039.safetensors",
-    "layers.26.self_attn.o_proj.weight": "model-00032-of-00039.safetensors",
-    "layers.26.self_attn.q_proj.weight": "model-00032-of-00039.safetensors",
-    "layers.26.self_attn.v_proj.weight": "model-00032-of-00039.safetensors",
-    "layers.27.block_sparse_moe.experts.0.w1.weight": "model-00033-of-00039.safetensors",
-    "layers.27.block_sparse_moe.experts.0.w2.weight": "model-00033-of-00039.safetensors",
-    "layers.27.block_sparse_moe.experts.0.w3.weight": "model-00033-of-00039.safetensors",
-    "layers.27.block_sparse_moe.experts.1.w1.weight": "model-00033-of-00039.safetensors",
-    "layers.27.block_sparse_moe.experts.1.w2.weight": "model-00033-of-00039.safetensors",
-    "layers.27.block_sparse_moe.experts.1.w3.weight": "model-00033-of-00039.safetensors",
-    "layers.27.block_sparse_moe.experts.2.w1.weight": "model-00033-of-00039.safetensors",
-    "layers.27.block_sparse_moe.experts.2.w2.weight": "model-00033-of-00039.safetensors",
-    "layers.27.block_sparse_moe.experts.2.w3.weight": "model-00033-of-00039.safetensors",
-    "layers.27.block_sparse_moe.experts.3.w1.weight": "model-00033-of-00039.safetensors",
-    "layers.27.block_sparse_moe.experts.3.w2.weight": "model-00033-of-00039.safetensors",
-    "layers.27.block_sparse_moe.experts.3.w3.weight": "model-00033-of-00039.safetensors",
-    "layers.27.block_sparse_moe.experts.4.w1.weight": "model-00033-of-00039.safetensors",
-    "layers.27.block_sparse_moe.experts.4.w2.weight": "model-00033-of-00039.safetensors",
-    "layers.27.block_sparse_moe.experts.4.w3.weight": "model-00033-of-00039.safetensors",
-    "layers.27.block_sparse_moe.experts.5.w1.weight": "model-00034-of-00039.safetensors",
-    "layers.27.block_sparse_moe.experts.5.w2.weight": "model-00034-of-00039.safetensors",
-    "layers.27.block_sparse_moe.experts.5.w3.weight": "model-00034-of-00039.safetensors",
-    "layers.27.block_sparse_moe.experts.6.w1.weight": "model-00034-of-00039.safetensors",
-    "layers.27.block_sparse_moe.experts.6.w2.weight": "model-00034-of-00039.safetensors",
-    "layers.27.block_sparse_moe.experts.6.w3.weight": "model-00034-of-00039.safetensors",
-    "layers.27.block_sparse_moe.experts.7.w1.weight": "model-00034-of-00039.safetensors",
-    "layers.27.block_sparse_moe.experts.7.w2.weight": "model-00034-of-00039.safetensors",
-    "layers.27.block_sparse_moe.experts.7.w3.weight": "model-00034-of-00039.safetensors",
-    "layers.27.block_sparse_moe.gate.weight": "model-00033-of-00039.safetensors",
-    "layers.27.input_layernorm.weight": "model-00034-of-00039.safetensors",
-    "layers.27.post_attention_layernorm.weight": "model-00034-of-00039.safetensors",
-    "layers.27.self_attn.k_proj.weight": "model-00033-of-00039.safetensors",
-    "layers.27.self_attn.o_proj.weight": "model-00033-of-00039.safetensors",
-    "layers.27.self_attn.q_proj.weight": "model-00033-of-00039.safetensors",
-    "layers.27.self_attn.v_proj.weight": "model-00033-of-00039.safetensors",
-    "layers.28.block_sparse_moe.experts.0.w1.weight": "model-00034-of-00039.safetensors",
-    "layers.28.block_sparse_moe.experts.0.w2.weight": "model-00034-of-00039.safetensors",
-    "layers.28.block_sparse_moe.experts.0.w3.weight": "model-00034-of-00039.safetensors",
-    "layers.28.block_sparse_moe.experts.1.w1.weight": "model-00034-of-00039.safetensors",
-    "layers.28.block_sparse_moe.experts.1.w2.weight": "model-00034-of-00039.safetensors",
-    "layers.28.block_sparse_moe.experts.1.w3.weight": "model-00034-of-00039.safetensors",
-    "layers.28.block_sparse_moe.experts.2.w1.weight": "model-00034-of-00039.safetensors",
-    "layers.28.block_sparse_moe.experts.2.w2.weight": "model-00034-of-00039.safetensors",
-    "layers.28.block_sparse_moe.experts.2.w3.weight": "model-00034-of-00039.safetensors",
-    "layers.28.block_sparse_moe.experts.3.w1.weight": "model-00034-of-00039.safetensors",
-    "layers.28.block_sparse_moe.experts.3.w2.weight": "model-00034-of-00039.safetensors",
-    "layers.28.block_sparse_moe.experts.3.w3.weight": "model-00035-of-00039.safetensors",
-    "layers.28.block_sparse_moe.experts.4.w1.weight": "model-00035-of-00039.safetensors",
-    "layers.28.block_sparse_moe.experts.4.w2.weight": "model-00035-of-00039.safetensors",
-    "layers.28.block_sparse_moe.experts.4.w3.weight": "model-00035-of-00039.safetensors",
-    "layers.28.block_sparse_moe.experts.5.w1.weight": "model-00035-of-00039.safetensors",
-    "layers.28.block_sparse_moe.experts.5.w2.weight": "model-00035-of-00039.safetensors",
-    "layers.28.block_sparse_moe.experts.5.w3.weight": "model-00035-of-00039.safetensors",
-    "layers.28.block_sparse_moe.experts.6.w1.weight": "model-00035-of-00039.safetensors",
-    "layers.28.block_sparse_moe.experts.6.w2.weight": "model-00035-of-00039.safetensors",
-    "layers.28.block_sparse_moe.experts.6.w3.weight": "model-00035-of-00039.safetensors",
-    "layers.28.block_sparse_moe.experts.7.w1.weight": "model-00035-of-00039.safetensors",
-    "layers.28.block_sparse_moe.experts.7.w2.weight": "model-00035-of-00039.safetensors",
-    "layers.28.block_sparse_moe.experts.7.w3.weight": "model-00035-of-00039.safetensors",
-    "layers.28.block_sparse_moe.gate.weight": "model-00034-of-00039.safetensors",
-    "layers.28.input_layernorm.weight": "model-00035-of-00039.safetensors",
-    "layers.28.post_attention_layernorm.weight": "model-00035-of-00039.safetensors",
-    "layers.28.self_attn.k_proj.weight": "model-00034-of-00039.safetensors",
-    "layers.28.self_attn.o_proj.weight": "model-00034-of-00039.safetensors",
-    "layers.28.self_attn.q_proj.weight": "model-00034-of-00039.safetensors",
-    "layers.28.self_attn.v_proj.weight": "model-00034-of-00039.safetensors",
-    "layers.29.block_sparse_moe.experts.0.w1.weight": "model-00035-of-00039.safetensors",
-    "layers.29.block_sparse_moe.experts.0.w2.weight": "model-00035-of-00039.safetensors",
-    "layers.29.block_sparse_moe.experts.0.w3.weight": "model-00035-of-00039.safetensors",
-    "layers.29.block_sparse_moe.experts.1.w1.weight": "model-00035-of-00039.safetensors",
-    "layers.29.block_sparse_moe.experts.1.w2.weight": "model-00035-of-00039.safetensors",
-    "layers.29.block_sparse_moe.experts.1.w3.weight": "model-00035-of-00039.safetensors",
-    "layers.29.block_sparse_moe.experts.2.w1.weight": "model-00035-of-00039.safetensors",
-    "layers.29.block_sparse_moe.experts.2.w2.weight": "model-00036-of-00039.safetensors",
-    "layers.29.block_sparse_moe.experts.2.w3.weight": "model-00036-of-00039.safetensors",
-    "layers.29.block_sparse_moe.experts.3.w1.weight": "model-00036-of-00039.safetensors",
-    "layers.29.block_sparse_moe.experts.3.w2.weight": "model-00036-of-00039.safetensors",
-    "layers.29.block_sparse_moe.experts.3.w3.weight": "model-00036-of-00039.safetensors",
-    "layers.29.block_sparse_moe.experts.4.w1.weight": "model-00036-of-00039.safetensors",
-    "layers.29.block_sparse_moe.experts.4.w2.weight": "model-00036-of-00039.safetensors",
-    "layers.29.block_sparse_moe.experts.4.w3.weight": "model-00036-of-00039.safetensors",
-    "layers.29.block_sparse_moe.experts.5.w1.weight": "model-00036-of-00039.safetensors",
-    "layers.29.block_sparse_moe.experts.5.w2.weight": "model-00036-of-00039.safetensors",
-    "layers.29.block_sparse_moe.experts.5.w3.weight": "model-00036-of-00039.safetensors",
-    "layers.29.block_sparse_moe.experts.6.w1.weight": "model-00036-of-00039.safetensors",
-    "layers.29.block_sparse_moe.experts.6.w2.weight": "model-00036-of-00039.safetensors",
-    "layers.29.block_sparse_moe.experts.6.w3.weight": "model-00036-of-00039.safetensors",
-    "layers.29.block_sparse_moe.experts.7.w1.weight": "model-00036-of-00039.safetensors",
-    "layers.29.block_sparse_moe.experts.7.w2.weight": "model-00036-of-00039.safetensors",
-    "layers.29.block_sparse_moe.experts.7.w3.weight": "model-00036-of-00039.safetensors",
-    "layers.29.block_sparse_moe.gate.weight": "model-00035-of-00039.safetensors",
-    "layers.29.input_layernorm.weight": "model-00036-of-00039.safetensors",
-    "layers.29.post_attention_layernorm.weight": "model-00036-of-00039.safetensors",
-    "layers.29.self_attn.k_proj.weight": "model-00035-of-00039.safetensors",
-    "layers.29.self_attn.o_proj.weight": "model-00035-of-00039.safetensors",
-    "layers.29.self_attn.q_proj.weight": "model-00035-of-00039.safetensors",
-    "layers.29.self_attn.v_proj.weight": "model-00035-of-00039.safetensors",
-    "layers.3.block_sparse_moe.experts.0.w1.weight": "model-00004-of-00039.safetensors",
-    "layers.3.block_sparse_moe.experts.0.w2.weight": "model-00004-of-00039.safetensors",
-    "layers.3.block_sparse_moe.experts.0.w3.weight": "model-00004-of-00039.safetensors",
-    "layers.3.block_sparse_moe.experts.1.w1.weight": "model-00004-of-00039.safetensors",
-    "layers.3.block_sparse_moe.experts.1.w2.weight": "model-00004-of-00039.safetensors",
-    "layers.3.block_sparse_moe.experts.1.w3.weight": "model-00004-of-00039.safetensors",
-    "layers.3.block_sparse_moe.experts.2.w1.weight": "model-00005-of-00039.safetensors",
-    "layers.3.block_sparse_moe.experts.2.w2.weight": "model-00005-of-00039.safetensors",
-    "layers.3.block_sparse_moe.experts.2.w3.weight": "model-00005-of-00039.safetensors",
-    "layers.3.block_sparse_moe.experts.3.w1.weight": "model-00005-of-00039.safetensors",
-    "layers.3.block_sparse_moe.experts.3.w2.weight": "model-00005-of-00039.safetensors",
-    "layers.3.block_sparse_moe.experts.3.w3.weight": "model-00005-of-00039.safetensors",
-    "layers.3.block_sparse_moe.experts.4.w1.weight": "model-00005-of-00039.safetensors",
-    "layers.3.block_sparse_moe.experts.4.w2.weight": "model-00005-of-00039.safetensors",
-    "layers.3.block_sparse_moe.experts.4.w3.weight": "model-00005-of-00039.safetensors",
-    "layers.3.block_sparse_moe.experts.5.w1.weight": "model-00005-of-00039.safetensors",
-    "layers.3.block_sparse_moe.experts.5.w2.weight": "model-00005-of-00039.safetensors",
-    "layers.3.block_sparse_moe.experts.5.w3.weight": "model-00005-of-00039.safetensors",
-    "layers.3.block_sparse_moe.experts.6.w1.weight": "model-00005-of-00039.safetensors",
-    "layers.3.block_sparse_moe.experts.6.w2.weight": "model-00005-of-00039.safetensors",
-    "layers.3.block_sparse_moe.experts.6.w3.weight": "model-00005-of-00039.safetensors",
-    "layers.3.block_sparse_moe.experts.7.w1.weight": "model-00005-of-00039.safetensors",
-    "layers.3.block_sparse_moe.experts.7.w2.weight": "model-00005-of-00039.safetensors",
-    "layers.3.block_sparse_moe.experts.7.w3.weight": "model-00005-of-00039.safetensors",
-    "layers.3.block_sparse_moe.gate.weight": "model-00004-of-00039.safetensors",
-    "layers.3.input_layernorm.weight": "model-00005-of-00039.safetensors",
-    "layers.3.post_attention_layernorm.weight": "model-00005-of-00039.safetensors",
-    "layers.3.self_attn.k_proj.weight": "model-00004-of-00039.safetensors",
-    "layers.3.self_attn.o_proj.weight": "model-00004-of-00039.safetensors",
-    "layers.3.self_attn.q_proj.weight": "model-00004-of-00039.safetensors",
-    "layers.3.self_attn.v_proj.weight": "model-00004-of-00039.safetensors",
-    "layers.30.block_sparse_moe.experts.0.w1.weight": "model-00036-of-00039.safetensors",
-    "layers.30.block_sparse_moe.experts.0.w2.weight": "model-00036-of-00039.safetensors",
-    "layers.30.block_sparse_moe.experts.0.w3.weight": "model-00036-of-00039.safetensors",
-    "layers.30.block_sparse_moe.experts.1.w1.weight": "model-00037-of-00039.safetensors",
-    "layers.30.block_sparse_moe.experts.1.w2.weight": "model-00037-of-00039.safetensors",
-    "layers.30.block_sparse_moe.experts.1.w3.weight": "model-00037-of-00039.safetensors",
-    "layers.30.block_sparse_moe.experts.2.w1.weight": "model-00037-of-00039.safetensors",
-    "layers.30.block_sparse_moe.experts.2.w2.weight": "model-00037-of-00039.safetensors",
-    "layers.30.block_sparse_moe.experts.2.w3.weight": "model-00037-of-00039.safetensors",
-    "layers.30.block_sparse_moe.experts.3.w1.weight": "model-00037-of-00039.safetensors",
-    "layers.30.block_sparse_moe.experts.3.w2.weight": "model-00037-of-00039.safetensors",
-    "layers.30.block_sparse_moe.experts.3.w3.weight": "model-00037-of-00039.safetensors",
-    "layers.30.block_sparse_moe.experts.4.w1.weight": "model-00037-of-00039.safetensors",
-    "layers.30.block_sparse_moe.experts.4.w2.weight": "model-00037-of-00039.safetensors",
-    "layers.30.block_sparse_moe.experts.4.w3.weight": "model-00037-of-00039.safetensors",
-    "layers.30.block_sparse_moe.experts.5.w1.weight": "model-00037-of-00039.safetensors",
-    "layers.30.block_sparse_moe.experts.5.w2.weight": "model-00037-of-00039.safetensors",
-    "layers.30.block_sparse_moe.experts.5.w3.weight": "model-00037-of-00039.safetensors",
-    "layers.30.block_sparse_moe.experts.6.w1.weight": "model-00037-of-00039.safetensors",
-    "layers.30.block_sparse_moe.experts.6.w2.weight": "model-00037-of-00039.safetensors",
-    "layers.30.block_sparse_moe.experts.6.w3.weight": "model-00037-of-00039.safetensors",
-    "layers.30.block_sparse_moe.experts.7.w1.weight": "model-00037-of-00039.safetensors",
-    "layers.30.block_sparse_moe.experts.7.w2.weight": "model-00037-of-00039.safetensors",
-    "layers.30.block_sparse_moe.experts.7.w3.weight": "model-00037-of-00039.safetensors",
-    "layers.30.block_sparse_moe.gate.weight": "model-00036-of-00039.safetensors",
-    "layers.30.input_layernorm.weight": "model-00037-of-00039.safetensors",
-    "layers.30.post_attention_layernorm.weight": "model-00037-of-00039.safetensors",
-    "layers.30.self_attn.k_proj.weight": "model-00036-of-00039.safetensors",
-    "layers.30.self_attn.o_proj.weight": "model-00036-of-00039.safetensors",
-    "layers.30.self_attn.q_proj.weight": "model-00036-of-00039.safetensors",
-    "layers.30.self_attn.v_proj.weight": "model-00036-of-00039.safetensors",
-    "layers.31.block_sparse_moe.experts.0.w1.weight": "model-00038-of-00039.safetensors",
-    "layers.31.block_sparse_moe.experts.0.w2.weight": "model-00038-of-00039.safetensors",
-    "layers.31.block_sparse_moe.experts.0.w3.weight": "model-00038-of-00039.safetensors",
-    "layers.31.block_sparse_moe.experts.1.w1.weight": "model-00038-of-00039.safetensors",
-    "layers.31.block_sparse_moe.experts.1.w2.weight": "model-00038-of-00039.safetensors",
-    "layers.31.block_sparse_moe.experts.1.w3.weight": "model-00038-of-00039.safetensors",
-    "layers.31.block_sparse_moe.experts.2.w1.weight": "model-00038-of-00039.safetensors",
-    "layers.31.block_sparse_moe.experts.2.w2.weight": "model-00038-of-00039.safetensors",
-    "layers.31.block_sparse_moe.experts.2.w3.weight": "model-00038-of-00039.safetensors",
-    "layers.31.block_sparse_moe.experts.3.w1.weight": "model-00038-of-00039.safetensors",
-    "layers.31.block_sparse_moe.experts.3.w2.weight": "model-00038-of-00039.safetensors",
-    "layers.31.block_sparse_moe.experts.3.w3.weight": "model-00038-of-00039.safetensors",
-    "layers.31.block_sparse_moe.experts.4.w1.weight": "model-00038-of-00039.safetensors",
-    "layers.31.block_sparse_moe.experts.4.w2.weight": "model-00038-of-00039.safetensors",
-    "layers.31.block_sparse_moe.experts.4.w3.weight": "model-00038-of-00039.safetensors",
-    "layers.31.block_sparse_moe.experts.5.w1.weight": "model-00038-of-00039.safetensors",
-    "layers.31.block_sparse_moe.experts.5.w2.weight": "model-00038-of-00039.safetensors",
-    "layers.31.block_sparse_moe.experts.5.w3.weight": "model-00038-of-00039.safetensors",
-    "layers.31.block_sparse_moe.experts.6.w1.weight": "model-00038-of-00039.safetensors",
-    "layers.31.block_sparse_moe.experts.6.w2.weight": "model-00038-of-00039.safetensors",
-    "layers.31.block_sparse_moe.experts.6.w3.weight": "model-00039-of-00039.safetensors",
-    "layers.31.block_sparse_moe.experts.7.w1.weight": "model-00039-of-00039.safetensors",
-    "layers.31.block_sparse_moe.experts.7.w2.weight": "model-00039-of-00039.safetensors",
-    "layers.31.block_sparse_moe.experts.7.w3.weight": "model-00039-of-00039.safetensors",
-    "layers.31.block_sparse_moe.gate.weight": "model-00038-of-00039.safetensors",
-    "layers.31.input_layernorm.weight": "model-00039-of-00039.safetensors",
-    "layers.31.post_attention_layernorm.weight": "model-00039-of-00039.safetensors",
-    "layers.31.self_attn.k_proj.weight": "model-00038-of-00039.safetensors",
-    "layers.31.self_attn.o_proj.weight": "model-00038-of-00039.safetensors",
-    "layers.31.self_attn.q_proj.weight": "model-00037-of-00039.safetensors",
-    "layers.31.self_attn.v_proj.weight": "model-00038-of-00039.safetensors",
-    "layers.4.block_sparse_moe.experts.0.w1.weight": "model-00005-of-00039.safetensors",
-    "layers.4.block_sparse_moe.experts.0.w2.weight": "model-00005-of-00039.safetensors",
-    "layers.4.block_sparse_moe.experts.0.w3.weight": "model-00006-of-00039.safetensors",
-    "layers.4.block_sparse_moe.experts.1.w1.weight": "model-00006-of-00039.safetensors",
-    "layers.4.block_sparse_moe.experts.1.w2.weight": "model-00006-of-00039.safetensors",
-    "layers.4.block_sparse_moe.experts.1.w3.weight": "model-00006-of-00039.safetensors",
-    "layers.4.block_sparse_moe.experts.2.w1.weight": "model-00006-of-00039.safetensors",
-    "layers.4.block_sparse_moe.experts.2.w2.weight": "model-00006-of-00039.safetensors",
-    "layers.4.block_sparse_moe.experts.2.w3.weight": "model-00006-of-00039.safetensors",
-    "layers.4.block_sparse_moe.experts.3.w1.weight": "model-00006-of-00039.safetensors",
-    "layers.4.block_sparse_moe.experts.3.w2.weight": "model-00006-of-00039.safetensors",
-    "layers.4.block_sparse_moe.experts.3.w3.weight": "model-00006-of-00039.safetensors",
-    "layers.4.block_sparse_moe.experts.4.w1.weight": "model-00006-of-00039.safetensors",
-    "layers.4.block_sparse_moe.experts.4.w2.weight": "model-00006-of-00039.safetensors",
-    "layers.4.block_sparse_moe.experts.4.w3.weight": "model-00006-of-00039.safetensors",
-    "layers.4.block_sparse_moe.experts.5.w1.weight": "model-00006-of-00039.safetensors",
-    "layers.4.block_sparse_moe.experts.5.w2.weight": "model-00006-of-00039.safetensors",
-    "layers.4.block_sparse_moe.experts.5.w3.weight": "model-00006-of-00039.safetensors",
-    "layers.4.block_sparse_moe.experts.6.w1.weight": "model-00006-of-00039.safetensors",
-    "layers.4.block_sparse_moe.experts.6.w2.weight": "model-00006-of-00039.safetensors",
-    "layers.4.block_sparse_moe.experts.6.w3.weight": "model-00006-of-00039.safetensors",
-    "layers.4.block_sparse_moe.experts.7.w1.weight": "model-00006-of-00039.safetensors",
-    "layers.4.block_sparse_moe.experts.7.w2.weight": "model-00006-of-00039.safetensors",
-    "layers.4.block_sparse_moe.experts.7.w3.weight": "model-00007-of-00039.safetensors",
-    "layers.4.block_sparse_moe.gate.weight": "model-00005-of-00039.safetensors",
-    "layers.4.input_layernorm.weight": "model-00007-of-00039.safetensors",
-    "layers.4.post_attention_layernorm.weight": "model-00007-of-00039.safetensors",
-    "layers.4.self_attn.k_proj.weight": "model-00005-of-00039.safetensors",
-    "layers.4.self_attn.o_proj.weight": "model-00005-of-00039.safetensors",
-    "layers.4.self_attn.q_proj.weight": "model-00005-of-00039.safetensors",
-    "layers.4.self_attn.v_proj.weight": "model-00005-of-00039.safetensors",
-    "layers.5.block_sparse_moe.experts.0.w1.weight": "model-00007-of-00039.safetensors",
-    "layers.5.block_sparse_moe.experts.0.w2.weight": "model-00007-of-00039.safetensors",
-    "layers.5.block_sparse_moe.experts.0.w3.weight": "model-00007-of-00039.safetensors",
-    "layers.5.block_sparse_moe.experts.1.w1.weight": "model-00007-of-00039.safetensors",
-    "layers.5.block_sparse_moe.experts.1.w2.weight": "model-00007-of-00039.safetensors",
-    "layers.5.block_sparse_moe.experts.1.w3.weight": "model-00007-of-00039.safetensors",
-    "layers.5.block_sparse_moe.experts.2.w1.weight": "model-00007-of-00039.safetensors",
-    "layers.5.block_sparse_moe.experts.2.w2.weight": "model-00007-of-00039.safetensors",
-    "layers.5.block_sparse_moe.experts.2.w3.weight": "model-00007-of-00039.safetensors",
-    "layers.5.block_sparse_moe.experts.3.w1.weight": "model-00007-of-00039.safetensors",
-    "layers.5.block_sparse_moe.experts.3.w2.weight": "model-00007-of-00039.safetensors",
-    "layers.5.block_sparse_moe.experts.3.w3.weight": "model-00007-of-00039.safetensors",
-    "layers.5.block_sparse_moe.experts.4.w1.weight": "model-00007-of-00039.safetensors",
-    "layers.5.block_sparse_moe.experts.4.w2.weight": "model-00007-of-00039.safetensors",
-    "layers.5.block_sparse_moe.experts.4.w3.weight": "model-00007-of-00039.safetensors",
-    "layers.5.block_sparse_moe.experts.5.w1.weight": "model-00007-of-00039.safetensors",
-    "layers.5.block_sparse_moe.experts.5.w2.weight": "model-00007-of-00039.safetensors",
-    "layers.5.block_sparse_moe.experts.5.w3.weight": "model-00007-of-00039.safetensors",
-    "layers.5.block_sparse_moe.experts.6.w1.weight": "model-00007-of-00039.safetensors",
-    "layers.5.block_sparse_moe.experts.6.w2.weight": "model-00008-of-00039.safetensors",
-    "layers.5.block_sparse_moe.experts.6.w3.weight": "model-00008-of-00039.safetensors",
-    "layers.5.block_sparse_moe.experts.7.w1.weight": "model-00008-of-00039.safetensors",
-    "layers.5.block_sparse_moe.experts.7.w2.weight": "model-00008-of-00039.safetensors",
-    "layers.5.block_sparse_moe.experts.7.w3.weight": "model-00008-of-00039.safetensors",
-    "layers.5.block_sparse_moe.gate.weight": "model-00007-of-00039.safetensors",
-    "layers.5.input_layernorm.weight": "model-00008-of-00039.safetensors",
-    "layers.5.post_attention_layernorm.weight": "model-00008-of-00039.safetensors",
-    "layers.5.self_attn.k_proj.weight": "model-00007-of-00039.safetensors",
-    "layers.5.self_attn.o_proj.weight": "model-00007-of-00039.safetensors",
-    "layers.5.self_attn.q_proj.weight": "model-00007-of-00039.safetensors",
-    "layers.5.self_attn.v_proj.weight": "model-00007-of-00039.safetensors",
-    "layers.6.block_sparse_moe.experts.0.w1.weight": "model-00008-of-00039.safetensors",
-    "layers.6.block_sparse_moe.experts.0.w2.weight": "model-00008-of-00039.safetensors",
-    "layers.6.block_sparse_moe.experts.0.w3.weight": "model-00008-of-00039.safetensors",
-    "layers.6.block_sparse_moe.experts.1.w1.weight": "model-00008-of-00039.safetensors",
-    "layers.6.block_sparse_moe.experts.1.w2.weight": "model-00008-of-00039.safetensors",
-    "layers.6.block_sparse_moe.experts.1.w3.weight": "model-00008-of-00039.safetensors",
-    "layers.6.block_sparse_moe.experts.2.w1.weight": "model-00008-of-00039.safetensors",
-    "layers.6.block_sparse_moe.experts.2.w2.weight": "model-00008-of-00039.safetensors",
-    "layers.6.block_sparse_moe.experts.2.w3.weight": "model-00008-of-00039.safetensors",
-    "layers.6.block_sparse_moe.experts.3.w1.weight": "model-00008-of-00039.safetensors",
-    "layers.6.block_sparse_moe.experts.3.w2.weight": "model-00008-of-00039.safetensors",
-    "layers.6.block_sparse_moe.experts.3.w3.weight": "model-00008-of-00039.safetensors",
-    "layers.6.block_sparse_moe.experts.4.w1.weight": "model-00008-of-00039.safetensors",
-    "layers.6.block_sparse_moe.experts.4.w2.weight": "model-00008-of-00039.safetensors",
-    "layers.6.block_sparse_moe.experts.4.w3.weight": "model-00008-of-00039.safetensors",
-    "layers.6.block_sparse_moe.experts.5.w1.weight": "model-00009-of-00039.safetensors",
-    "layers.6.block_sparse_moe.experts.5.w2.weight": "model-00009-of-00039.safetensors",
-    "layers.6.block_sparse_moe.experts.5.w3.weight": "model-00009-of-00039.safetensors",
-    "layers.6.block_sparse_moe.experts.6.w1.weight": "model-00009-of-00039.safetensors",
-    "layers.6.block_sparse_moe.experts.6.w2.weight": "model-00009-of-00039.safetensors",
-    "layers.6.block_sparse_moe.experts.6.w3.weight": "model-00009-of-00039.safetensors",
-    "layers.6.block_sparse_moe.experts.7.w1.weight": "model-00009-of-00039.safetensors",
-    "layers.6.block_sparse_moe.experts.7.w2.weight": "model-00009-of-00039.safetensors",
-    "layers.6.block_sparse_moe.experts.7.w3.weight": "model-00009-of-00039.safetensors",
-    "layers.6.block_sparse_moe.gate.weight": "model-00008-of-00039.safetensors",
-    "layers.6.input_layernorm.weight": "model-00009-of-00039.safetensors",
-    "layers.6.post_attention_layernorm.weight": "model-00009-of-00039.safetensors",
-    "layers.6.self_attn.k_proj.weight": "model-00008-of-00039.safetensors",
-    "layers.6.self_attn.o_proj.weight": "model-00008-of-00039.safetensors",
-    "layers.6.self_attn.q_proj.weight": "model-00008-of-00039.safetensors",
-    "layers.6.self_attn.v_proj.weight": "model-00008-of-00039.safetensors",
-    "layers.7.block_sparse_moe.experts.0.w1.weight": "model-00009-of-00039.safetensors",
-    "layers.7.block_sparse_moe.experts.0.w2.weight": "model-00009-of-00039.safetensors",
-    "layers.7.block_sparse_moe.experts.0.w3.weight": "model-00009-of-00039.safetensors",
-    "layers.7.block_sparse_moe.experts.1.w1.weight": "model-00009-of-00039.safetensors",
-    "layers.7.block_sparse_moe.experts.1.w2.weight": "model-00009-of-00039.safetensors",
-    "layers.7.block_sparse_moe.experts.1.w3.weight": "model-00009-of-00039.safetensors",
-    "layers.7.block_sparse_moe.experts.2.w1.weight": "model-00009-of-00039.safetensors",
-    "layers.7.block_sparse_moe.experts.2.w2.weight": "model-00009-of-00039.safetensors",
-    "layers.7.block_sparse_moe.experts.2.w3.weight": "model-00009-of-00039.safetensors",
-    "layers.7.block_sparse_moe.experts.3.w1.weight": "model-00009-of-00039.safetensors",
-    "layers.7.block_sparse_moe.experts.3.w2.weight": "model-00009-of-00039.safetensors",
-    "layers.7.block_sparse_moe.experts.3.w3.weight": "model-00010-of-00039.safetensors",
-    "layers.7.block_sparse_moe.experts.4.w1.weight": "model-00010-of-00039.safetensors",
-    "layers.7.block_sparse_moe.experts.4.w2.weight": "model-00010-of-00039.safetensors",
-    "layers.7.block_sparse_moe.experts.4.w3.weight": "model-00010-of-00039.safetensors",
-    "layers.7.block_sparse_moe.experts.5.w1.weight": "model-00010-of-00039.safetensors",
-    "layers.7.block_sparse_moe.experts.5.w2.weight": "model-00010-of-00039.safetensors",
-    "layers.7.block_sparse_moe.experts.5.w3.weight": "model-00010-of-00039.safetensors",
-    "layers.7.block_sparse_moe.experts.6.w1.weight": "model-00010-of-00039.safetensors",
-    "layers.7.block_sparse_moe.experts.6.w2.weight": "model-00010-of-00039.safetensors",
-    "layers.7.block_sparse_moe.experts.6.w3.weight": "model-00010-of-00039.safetensors",
-    "layers.7.block_sparse_moe.experts.7.w1.weight": "model-00010-of-00039.safetensors",
-    "layers.7.block_sparse_moe.experts.7.w2.weight": "model-00010-of-00039.safetensors",
-    "layers.7.block_sparse_moe.experts.7.w3.weight": "model-00010-of-00039.safetensors",
-    "layers.7.block_sparse_moe.gate.weight": "model-00009-of-00039.safetensors",
-    "layers.7.input_layernorm.weight": "model-00010-of-00039.safetensors",
-    "layers.7.post_attention_layernorm.weight": "model-00010-of-00039.safetensors",
-    "layers.7.self_attn.k_proj.weight": "model-00009-of-00039.safetensors",
-    "layers.7.self_attn.o_proj.weight": "model-00009-of-00039.safetensors",
-    "layers.7.self_attn.q_proj.weight": "model-00009-of-00039.safetensors",
-    "layers.7.self_attn.v_proj.weight": "model-00009-of-00039.safetensors",
-    "layers.8.block_sparse_moe.experts.0.w1.weight": "model-00010-of-00039.safetensors",
-    "layers.8.block_sparse_moe.experts.0.w2.weight": "model-00010-of-00039.safetensors",
-    "layers.8.block_sparse_moe.experts.0.w3.weight": "model-00010-of-00039.safetensors",
-    "layers.8.block_sparse_moe.experts.1.w1.weight": "model-00010-of-00039.safetensors",
-    "layers.8.block_sparse_moe.experts.1.w2.weight": "model-00010-of-00039.safetensors",
-    "layers.8.block_sparse_moe.experts.1.w3.weight": "model-00010-of-00039.safetensors",
-    "layers.8.block_sparse_moe.experts.2.w1.weight": "model-00010-of-00039.safetensors",
-    "layers.8.block_sparse_moe.experts.2.w2.weight": "model-00011-of-00039.safetensors",
-    "layers.8.block_sparse_moe.experts.2.w3.weight": "model-00011-of-00039.safetensors",
-    "layers.8.block_sparse_moe.experts.3.w1.weight": "model-00011-of-00039.safetensors",
-    "layers.8.block_sparse_moe.experts.3.w2.weight": "model-00011-of-00039.safetensors",
-    "layers.8.block_sparse_moe.experts.3.w3.weight": "model-00011-of-00039.safetensors",
-    "layers.8.block_sparse_moe.experts.4.w1.weight": "model-00011-of-00039.safetensors",
-    "layers.8.block_sparse_moe.experts.4.w2.weight": "model-00011-of-00039.safetensors",
-    "layers.8.block_sparse_moe.experts.4.w3.weight": "model-00011-of-00039.safetensors",
-    "layers.8.block_sparse_moe.experts.5.w1.weight": "model-00011-of-00039.safetensors",
-    "layers.8.block_sparse_moe.experts.5.w2.weight": "model-00011-of-00039.safetensors",
-    "layers.8.block_sparse_moe.experts.5.w3.weight": "model-00011-of-00039.safetensors",
-    "layers.8.block_sparse_moe.experts.6.w1.weight": "model-00011-of-00039.safetensors",
-    "layers.8.block_sparse_moe.experts.6.w2.weight": "model-00011-of-00039.safetensors",
-    "layers.8.block_sparse_moe.experts.6.w3.weight": "model-00011-of-00039.safetensors",
-    "layers.8.block_sparse_moe.experts.7.w1.weight": "model-00011-of-00039.safetensors",
-    "layers.8.block_sparse_moe.experts.7.w2.weight": "model-00011-of-00039.safetensors",
-    "layers.8.block_sparse_moe.experts.7.w3.weight": "model-00011-of-00039.safetensors",
-    "layers.8.block_sparse_moe.gate.weight": "model-00010-of-00039.safetensors",
-    "layers.8.input_layernorm.weight": "model-00011-of-00039.safetensors",
-    "layers.8.post_attention_layernorm.weight": "model-00011-of-00039.safetensors",
-    "layers.8.self_attn.k_proj.weight": "model-00010-of-00039.safetensors",
-    "layers.8.self_attn.o_proj.weight": "model-00010-of-00039.safetensors",
-    "layers.8.self_attn.q_proj.weight": "model-00010-of-00039.safetensors",
-    "layers.8.self_attn.v_proj.weight": "model-00010-of-00039.safetensors",
-    "layers.9.block_sparse_moe.experts.0.w1.weight": "model-00011-of-00039.safetensors",
-    "layers.9.block_sparse_moe.experts.0.w2.weight": "model-00011-of-00039.safetensors",
-    "layers.9.block_sparse_moe.experts.0.w3.weight": "model-00011-of-00039.safetensors",
-    "layers.9.block_sparse_moe.experts.1.w1.weight": "model-00012-of-00039.safetensors",
-    "layers.9.block_sparse_moe.experts.1.w2.weight": "model-00012-of-00039.safetensors",
-    "layers.9.block_sparse_moe.experts.1.w3.weight": "model-00012-of-00039.safetensors",
-    "layers.9.block_sparse_moe.experts.2.w1.weight": "model-00012-of-00039.safetensors",
-    "layers.9.block_sparse_moe.experts.2.w2.weight": "model-00012-of-00039.safetensors",
-    "layers.9.block_sparse_moe.experts.2.w3.weight": "model-00012-of-00039.safetensors",
-    "layers.9.block_sparse_moe.experts.3.w1.weight": "model-00012-of-00039.safetensors",
-    "layers.9.block_sparse_moe.experts.3.w2.weight": "model-00012-of-00039.safetensors",
-    "layers.9.block_sparse_moe.experts.3.w3.weight": "model-00012-of-00039.safetensors",
-    "layers.9.block_sparse_moe.experts.4.w1.weight": "model-00012-of-00039.safetensors",
-    "layers.9.block_sparse_moe.experts.4.w2.weight": "model-00012-of-00039.safetensors",
-    "layers.9.block_sparse_moe.experts.4.w3.weight": "model-00012-of-00039.safetensors",
-    "layers.9.block_sparse_moe.experts.5.w1.weight": "model-00012-of-00039.safetensors",
-    "layers.9.block_sparse_moe.experts.5.w2.weight": "model-00012-of-00039.safetensors",
-    "layers.9.block_sparse_moe.experts.5.w3.weight": "model-00012-of-00039.safetensors",
-    "layers.9.block_sparse_moe.experts.6.w1.weight": "model-00012-of-00039.safetensors",
-    "layers.9.block_sparse_moe.experts.6.w2.weight": "model-00012-of-00039.safetensors",
-    "layers.9.block_sparse_moe.experts.6.w3.weight": "model-00012-of-00039.safetensors",
-    "layers.9.block_sparse_moe.experts.7.w1.weight": "model-00012-of-00039.safetensors",
-    "layers.9.block_sparse_moe.experts.7.w2.weight": "model-00012-of-00039.safetensors",
-    "layers.9.block_sparse_moe.experts.7.w3.weight": "model-00012-of-00039.safetensors",
-    "layers.9.block_sparse_moe.gate.weight": "model-00011-of-00039.safetensors",
-    "layers.9.input_layernorm.weight": "model-00012-of-00039.safetensors",
-    "layers.9.post_attention_layernorm.weight": "model-00012-of-00039.safetensors",
-    "layers.9.self_attn.k_proj.weight": "model-00011-of-00039.safetensors",
-    "layers.9.self_attn.o_proj.weight": "model-00011-of-00039.safetensors",
-    "layers.9.self_attn.q_proj.weight": "model-00011-of-00039.safetensors",
-    "layers.9.self_attn.v_proj.weight": "model-00011-of-00039.safetensors",
-    "norm.weight": "model-00039-of-00039.safetensors"
-  }
-}

special_tokens_map.json

@@ -1,30 +0,0 @@
-{
-  "bos_token": {
-    "content": "<s>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "eos_token": {
-    "content": "</s>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "pad_token": {
-    "content": "</s>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "unk_token": {
-    "content": "<unk>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  }
-}

tokenizer.model

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055
-size 493443

tokenizer_config.json

@@ -1,45 +0,0 @@
-{
-  "add_bos_token": true,
-  "add_eos_token": false,
-  "added_tokens_decoder": {
-    "0": {
-      "content": "<unk>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "1": {
-      "content": "<s>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "2": {
-      "content": "</s>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    }
-  },
-  "additional_special_tokens": [],
-  "bos_token": "<s>",
-  "chat_template": "{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if message['role'] == 'user' %}{{ '[INST] ' + message['content'] + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ message['content'] + eos_token}}{% else %}{{ raise_exception('Only user and assistant roles are supported!') }}{% endif %}{% endfor %}",
-  "clean_up_tokenization_spaces": false,
-  "eos_token": "</s>",
-  "legacy": true,
-  "model_max_length": 1000000000000000019884624838656,
-  "pad_token": "</s>",
-  "sp_model_kwargs": {},
-  "spaces_between_special_tokens": false,
-  "tokenizer_class": "LlamaTokenizer",
-  "trust_remote_code": true,
-  "unk_token": "<unk>",
-  "use_default_system_prompt": false,
-  "use_fast": true
-}