new model

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.json filter=lfs diff=lfs merge=lfs -text

config.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eff7add3ca04269e7341d69d1938df15c3408fdf9de227291ef809164f891408
+size 897

configuration_cohere.py

@@ -0,0 +1,159 @@
+# coding=utf-8
+# Copyright 2024 Cohere team. All rights reserved.
+#
+# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
+# and OPT implementations in this library. It has been modified from its
+# original forms to accommodate minor architectural differences compared
+# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" Cohere model configuration"""
+
+from transformers import PretrainedConfig, AutoConfig
+from transformers.utils import logging
+
+
+logger = logging.get_logger(__name__)
+
+
+class CohereConfig(PretrainedConfig):
+    r"""
+    This is the configuration class to store the configuration of a [`CohereModel`]. It is used to instantiate an Cohere
+    model according to the specified arguments, defining the model architecture.
+
+    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+    documentation from [`PretrainedConfig`] for more information.
+
+
+    Args:
+        vocab_size (`int`, *optional*, defaults to 256000):
+            Vocabulary size of the Cohere model. Defines the number of different tokens that can be represented by the
+            `inputs_ids` passed when calling [`CohereModel`]
+        hidden_size (`int`, *optional*, defaults to 8192):
+            Dimension of the hidden representations.
+        intermediate_size (`int`, *optional*, defaults to 22528):
+            Dimension of the MLP representations.
+        num_hidden_layers (`int`, *optional*, defaults to 40):
+            Number of hidden layers in the Transformer decoder.
+        num_attention_heads (`int`, *optional*, defaults to 64):
+            Number of attention heads for each attention layer in the Transformer decoder.
+        num_key_value_heads (`int`, *optional*):
+            This is the number of key_value heads that should be used to implement Grouped Query Attention. If
+            `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
+            `num_key_value_heads=1 the model will use Multi Query Attention (MQA) otherwise GQA is used. When
+            converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
+            by meanpooling all the original heads within that group. For more details checkout [this
+            paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to
+            `num_attention_heads`.
+        hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
+            The non-linear activation function (function or string) in the decoder.
+        max_position_embeddings (`int`, *optional*, defaults to 8192):
+            The maximum sequence length that this model might ever be used with. 
+        initializer_range (`float`, *optional*, defaults to 0.02):
+            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
+        layer_norm_eps (`float`, *optional*, defaults to 1e-05):
+            The epsilon used by the layer normalization.
+        use_cache (`bool`, *optional*, defaults to `True`):
+            Whether or not the model should return the last key/values attentions (not used by all models). Only
+            relevant if `config.is_decoder=True`.
+        pad_token_id (`int`, *optional*, defaults to 0):
+            Padding token id.
+        bos_token_id (`int`, *optional*, defaults to 5):
+            Beginning of stream token id.
+        eos_token_id (`int`, *optional*, defaults to 255001):
+            End of stream token id.
+        pretraining_tp (`int`, *optional*, defaults to 1):
+            Experimental feature. Tensor parallelism rank used during pretraining. Please refer to [this
+            document](https://huggingface.co/docs/transformers/main/perf_train_gpu_many#tensor-parallelism) to understand more about it. This value is
+            necessary to ensure exact reproducibility of the pretraining results. Please refer to [this
+            issue](https://github.com/pytorch/pytorch/issues/76232).
+        tie_word_embeddings (`bool`, *optional*, defaults to `False`):
+            Whether to tie weight embeddings
+        rope_theta (`float`, *optional*, defaults to 10000.0):
+            The base period of the RoPE embeddings.
+        attention_bias (`bool`, defaults to `False`, *optional*, defaults to `False`):
+            Whether to use a bias in the query, key, value and output projection layers during self-attention.
+        attention_dropout (`float`, *optional*, defaults to 0.0):
+            The dropout ratio for the attention probabilities.
+
+    ```python
+    >>> from transformers import CohereModel, CohereConfig
+
+    >>> # Initializing a Cohere model configuration
+    >>> configuration = CohereConfig()
+
+    >>> # Initializing a model from the Cohere configuration
+    >>> model = CohereModel(configuration)
+
+    >>> # Accessing the model configuration
+    >>> configuration = model.config
+    ```"""
+
+    model_type = "cohere"
+    keys_to_ignore_at_inference = ["past_key_values"]
+
+    def __init__(
+        self,
+        vocab_size=256000,
+        hidden_size=8192,
+        intermediate_size=22528,
+        num_hidden_layers=40,
+        num_attention_heads=64,
+        num_key_value_heads=None,
+        hidden_act="silu",
+        max_position_embeddings=8192,
+        initializer_range=0.02,
+        layer_norm_eps=1e-5,
+        use_cache=True,
+        pad_token_id=0,
+        bos_token_id=5,
+        eos_token_id=255001,
+        pretraining_tp=1,
+        tie_word_embeddings=True,
+        rope_theta=10000.0,
+        attention_bias=False,
+        attention_dropout=0.0,
+        **kwargs,
+    ):
+        self.vocab_size = vocab_size
+        self.max_position_embeddings = max_position_embeddings
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+
+        # for backward compatibility
+        if num_key_value_heads is None:
+            num_key_value_heads = num_attention_heads
+
+        self.num_key_value_heads = num_key_value_heads
+        self.hidden_act = hidden_act
+        self.initializer_range = initializer_range
+        self.layer_norm_eps = layer_norm_eps
+        self.pretraining_tp = pretraining_tp
+        self.use_cache = use_cache
+        self.rope_theta = rope_theta
+        self.attention_bias = attention_bias
+        self.attention_dropout = attention_dropout
+
+        super().__init__(
+            pad_token_id=pad_token_id,
+            bos_token_id=bos_token_id,
+            eos_token_id=eos_token_id,
+            tie_word_embeddings=tie_word_embeddings,
+            **kwargs,
+        )
+
+
+# register the model config to AutoConfig
+AutoConfig.register("cohere", CohereConfig)

generation_config.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:97e9138e5c4f5db123f5d9ff4b90a75d4ae33494e7f3032e1e0731dff3398ded
+size 137

model-00001-of-00015_chunk_1.txt.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:983010042bda97f1ebdcd90895ca25c55afda192e9d540b8c7e72016b0c256eb
+size 912917594

model-00001-of-00015_chunk_2.txt.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e725a13e9d0f3a957a8448326a1d1f2a7a12504fd3fbc03360d068c933bfe771
+size 922534840

model-00001-of-00015_chunk_3.txt.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:422c6afc3796db1bb70c38d32ef144e53077f1c9bd847bcb25360edc637e85ae
+size 922774556

model-00001-of-00015_chunk_4.txt.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:55728ffd8a46353c9e7c4db1be7793789bdfb9e1be04ad0b5969e20813f63235
+size 922926032

model-00001-of-00015_chunk_5.txt.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d472e88bece7d2d172e87e1d58e258a1e539624cfa1c4236da7b6db95a51155f
+size 685617441

model-00002-of-00015_chunk_1.txt.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3464637b32a2674adc814cf417f3c5e55a95449705cceace5b6e61af46be1187
+size 921808207

model-00002-of-00015_chunk_2.txt.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c78657ad4452cdce39deed161fa529eb7e175e0773908258ec7ef3a4040fb11
+size 922063330

model-00002-of-00015_chunk_3.txt.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8505f501b7ee4b0380891105106eaa02d75f93d769e4f8ff42bbab3072e86226
+size 922797635

model-00002-of-00015_chunk_4.txt.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f04833376a6e05c53fcb6addc722614de2f53faa06c1ab2f43bce22b5f3f6f00
+size 922060188

model-00002-of-00015_chunk_5.txt.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3725fd201b97180acb0372182014543f4a090a7691a684302662c4684f4daea8
+size 877819011

model-00003-of-00015_chunk_1.txt.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:96c5653194819a2bb83875d6c0cc8c68697816c656295e72d5f15fe545f75e78
+size 921724274

model-00003-of-00015_chunk_2.txt.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3115be698a9979a3ff89decc98564431bfaef6fb00eae80ee5f4c7bcb80b6da7
+size 921710520

model-00003-of-00015_chunk_3.txt.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:04e28eb2675940232ad8eade0a75ec924c9687d86d9bb2f12f63c809234ef632
+size 922036737

model-00003-of-00015_chunk_4.txt.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:18f0b64193486e8ab137dd4fda38339b9d14adf6e4d4ae709225619de4598a0a
+size 921878214

model-00003-of-00015_chunk_5.txt.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7cb25e3ae0d69ae72b87ea3184bc8e0ab3f495f20bb56ac81165495c2d34f0d3
+size 877644630

model-00004-of-00015_chunk_1.txt.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f4fb834c7e88ec7628d5129cab745fbeded42de3d73e44162923ae14f81bc3ff
+size 921773749

model-00004-of-00015_chunk_2.txt.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c7f6356b316c7cfa6db276194b4a5e1bc77ad0dfef8dd0384d96c8d8dd1c6ef5
+size 921796569

model-00004-of-00015_chunk_3.txt.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:59c4109a5c4e88b66094e37093971a228c1be8f4ef51456d4d42f90b57353ee3
+size 921958299

model-00004-of-00015_chunk_4.txt.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:81ed13c050507fe1c217b41bc1761046c721df2ed4b4e727e5dc8783534706f1
+size 921900645

model-00004-of-00015_chunk_5.txt.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:60f1ba1e97d3216b5c4b6d83cf575f64b57c124c14dfc2da62f09f1a103f6b39
+size 877441604

model-00005-of-00015_chunk_1.txt.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c2c5cf6309a834ce42625c3b54a80ac5bbff97e66df5e9bfdeb34b014490da39
+size 921786678

model-00005-of-00015_chunk_2.txt.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:42298cf52bbbedb30e592cafac2b40d3e3512f803a7ff83e53f805061f3ec55e
+size 921921877

model-00005-of-00015_chunk_3.txt.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:abb9e6726e057ff312869cee617dbc00f884ce59122d8c19f997d4d07a7785d1
+size 921887137

model-00005-of-00015_chunk_4.txt.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:95b16edde10e4dd06636aa2498d3d8f64d6b6099de5c08144440cce0ec9fcb71
+size 921955411

model-00005-of-00015_chunk_5.txt.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6e82120e90b92637e1045a9aa8cfb6ee1c9fffa4364936824dd52cc7f297d67b
+size 877233350

model-00006-of-00015_chunk_1.txt.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:97835a28fe018e053c3cafc65bf6743e9443c9de6f886c0b3432f8b38cb72cfa
+size 921788876

model-00006-of-00015_chunk_2.txt.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6cca17bfde243ef3804bcaa946f4a9695eeea20df0dfb84e5bda16d38b03beca
+size 921787566

model-00006-of-00015_chunk_3.txt.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:97f0e6614cfc6639557d34594e6ed5f61a952c8df8d42fe60606c1b758dc7186
+size 921880573

model-00006-of-00015_chunk_4.txt.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:54b9173b7306456925e85349df315d0443ddb222bc667c1996a13d0f81dd1b01
+size 921854378

model-00006-of-00015_chunk_5.txt.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:797b84e8fd00955605b3c7409225e6594c70fad4eb3df0befc0eb75f5d5f2d00
+size 877368625

model-00007-of-00015_chunk_1.txt.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fd85c74c23d8c7898e640aa827cb447fb277c26eb492ffb151d94f8cec727c90
+size 921789073

model-00007-of-00015_chunk_2.txt.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2fe5a306e1ac7f125d0bb62579e41a181dab6962f7f93db800be53e3146e21c7
+size 921797507

model-00007-of-00015_chunk_3.txt.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d7877b6297f67dc461e145b368d7be3f476c42ba61b5315edeedc544b98bca2e
+size 921871946

model-00007-of-00015_chunk_4.txt.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be834deda93b3f87b964b755776cf232f5fd717d8e5a70604ea075f9f12faa26
+size 921860093

model-00007-of-00015_chunk_5.txt.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f9041ba7416d206299650ceb4d3302c65fb2be921a5fe8b348ebeecc03cca6c4
+size 877358553

model-00008-of-00015_chunk_1.txt.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f76fb8cf074749d5fcb0670506ecba0e13086880e9d37aa8fd0a357af7cc8098
+size 921788298

model-00008-of-00015_chunk_2.txt.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:42f4b2d8645aca90276a63fc5d8b1fae3aa3357cca7d9dd5d4da27463a5857cc
+size 921777511

model-00008-of-00015_chunk_3.txt.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:216a87c1f22d85c3ec5d47b163a8f107a461e8849a14d4bf4c556f6809b23d63
+size 921837375

model-00008-of-00015_chunk_4.txt.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61c4dd0fd79173b4085ae02b3567dec5fc200f767ce2e66bf06f4c7665dd0b75
+size 921835697

model-00008-of-00015_chunk_5.txt.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4735cf0047ba7621ec7aed5d61b09bd381945b1bf947f7c6c770ad373d4478fa
+size 877304942

model-00009-of-00015_chunk_1.txt.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b5fce150e764ce27215846ad44f90eb7151441ad481eb3ac528b734ce693e5e
+size 921745778

model-00009-of-00015_chunk_2.txt.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eb558cb0deb80c17b100107e80bf2b2efee172b6ff57cc012bc830afecd740f8
+size 921739688

model-00009-of-00015_chunk_3.txt.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:376dbad9a1588cf295ea6389e2f6dec340473d358f1197dd096b14e6a98b8b85
+size 921798656

model-00009-of-00015_chunk_4.txt.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:995a0c275fa9fac9c067713295bf064ee0571a1a5b7594b2fe9a3a699ceb4c33
+size 921760560

model-00009-of-00015_chunk_5.txt.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2983d57c6d6a7286b45b47b9d85c493690dd737086af6b3b791827a0dc5291c1
+size 877254114

model-00010-of-00015_chunk_1.txt.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d6320f45db0e4f102ddaed958e335cbbe57229148add0665b999d89ab4379134
+size 921689805