Upload 10 files

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "NousResearch/Hermes-2-Pro-Llama-3-8B",
+  "_name_or_path": "OpenPipe/Hermes-2-Theta-Llama-3-8B-32k",
   "architectures": [
     "LlamaForCausalLM"
   ],
@@ -11,7 +11,7 @@
   "hidden_size": 4096,
   "initializer_range": 0.02,
   "intermediate_size": 14336,
-  "max_position_embeddings": 8192,
+  "max_position_embeddings": 32768,
   "model_type": "llama",
   "num_attention_heads": 32,
   "num_hidden_layers": 32,
@@ -19,11 +19,11 @@
   "pretraining_tp": 1,
   "rms_norm_eps": 1e-05,
   "rope_scaling": null,
-  "rope_theta": 500000.0,
+  "rope_theta": 4000000.0,
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
   "transformers_version": "4.40.1",
-  "unsloth_version": "2024.4",
-  "use_cache": false,
-  "vocab_size": 128288
+  "unsloth_version": "2024.5",
+  "use_cache": true,
+  "vocab_size": 128256
 }

generation_config.json

@@ -3,5 +3,6 @@
   "bos_token_id": 128000,
   "do_sample": true,
   "eos_token_id": 128003,
-  "transformers_version": "4.40.1"
+  "transformers_version": "4.40.1",
+  "use_cache": false
 }

model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fe75825dbc631635d71d22edb62fe9fef6231c95f66d3babf368f914b927abec
-size 4976960816
+oid sha256:b8dffb90c4b417d0bcb8d3a55857e329335b79a1051fcf6dad49316588402c9e
+size 4976698672

model-00002-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:24e4cb7372d79b0190639b78927cbf72553c6e56d89c5f0c916ba43c850fd257
+oid sha256:7d96bb3a3bb8590b5e48ca5816d06b77325ccbb32a75bbf9faebc18a193200e3
 size 4999802720

model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cd6d163c8689d9a342f9bf2f7873bb23144181c48eda39aa697843631701b833
+oid sha256:58bc6c3b51ed49eb46be9eabcf690223ab001bdfc4e9f90c7b59bb941137b7e5
 size 4915916176

model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:18c0d608d008cd477f7dad8894adc42d2c692dcbb22a6a58f847d3fa491771d8
-size 1168400952
+oid sha256:b30eda3412fce88c7bfbe08a46d0ec1d4f2662f5feb9b35de9fddb1e92c9dfd0
+size 1168138808

model.safetensors.index.json

@@ -1,6 +1,6 @@
 {
   "metadata": {
-    "total_size": 16061046784
+    "total_size": 16060522496
   },
   "weight_map": {
     "lm_head.weight": "model-00004-of-00004.safetensors",

tokenizer.json

@@ -86,7 +86,7 @@
     },
     {
       "id": 128009,
-      "content": "<|eot_id>",
+      "content": "<|eot_id|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -2306,294 +2306,6 @@
       "rstrip": false,
       "normalized": false,
       "special": true
-    },
-    {
-      "id": 128256,
-      "content": "<|eot_id|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 128257,
-      "content": "<|reserved_special_token_251|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 128258,
-      "content": "<|reserved_special_token_252|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 128259,
-      "content": "<|reserved_special_token_253|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 128260,
-      "content": "<|reserved_special_token_254|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 128261,
-      "content": "<|reserved_special_token_255|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 128262,
-      "content": "<|reserved_special_token_256|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 128263,
-      "content": "<|reserved_special_token_257|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 128264,
-      "content": "<|reserved_special_token_258|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 128265,
-      "content": "<|reserved_special_token_259|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 128266,
-      "content": "<|reserved_special_token_260|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 128267,
-      "content": "<|reserved_special_token_261|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 128268,
-      "content": "<|reserved_special_token_262|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 128269,
-      "content": "<|reserved_special_token_263|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 128270,
-      "content": "<|reserved_special_token_264|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 128271,
-      "content": "<|reserved_special_token_265|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 128272,
-      "content": "<|reserved_special_token_266|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 128273,
-      "content": "<|reserved_special_token_267|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 128274,
-      "content": "<|reserved_special_token_268|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 128275,
-      "content": "<|reserved_special_token_269|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 128276,
-      "content": "<|reserved_special_token_270|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 128277,
-      "content": "<|reserved_special_token_271|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 128278,
-      "content": "<|reserved_special_token_272|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 128279,
-      "content": "<|reserved_special_token_273|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 128280,
-      "content": "<|reserved_special_token_274|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 128281,
-      "content": "<|reserved_special_token_275|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 128282,
-      "content": "<|reserved_special_token_276|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 128283,
-      "content": "<|reserved_special_token_277|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 128284,
-      "content": "<|reserved_special_token_278|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 128285,
-      "content": "<|reserved_special_token_279|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 128286,
-      "content": "<|reserved_special_token_280|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 128287,
-      "content": "<|reserved_special_token_281|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
     }
   ],
   "normalizer": null,
@@ -2695,7 +2407,7 @@
     "end_of_word_suffix": null,
     "fuse_unk": false,
     "byte_fallback": false,
-    "ignore_merges": true,
+    "ignore_merges": false,
     "vocab": {
       "!": 0,
       "\"": 1,

tokenizer_config.json

@@ -73,7 +73,7 @@
       "special": false
     },
     "128009": {
-      "content": "<|eot_id>",
+      "content": "<|eot_id|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -2047,262 +2047,6 @@
       "rstrip": false,
       "single_word": false,
       "special": true
-    },
-    "128256": {
-      "content": "<|eot_id|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "128257": {
-      "content": "<|reserved_special_token_251|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "128258": {
-      "content": "<|reserved_special_token_252|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "128259": {
-      "content": "<|reserved_special_token_253|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "128260": {
-      "content": "<|reserved_special_token_254|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "128261": {
-      "content": "<|reserved_special_token_255|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "128262": {
-      "content": "<|reserved_special_token_256|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "128263": {
-      "content": "<|reserved_special_token_257|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "128264": {
-      "content": "<|reserved_special_token_258|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "128265": {
-      "content": "<|reserved_special_token_259|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "128266": {
-      "content": "<|reserved_special_token_260|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "128267": {
-      "content": "<|reserved_special_token_261|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "128268": {
-      "content": "<|reserved_special_token_262|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "128269": {
-      "content": "<|reserved_special_token_263|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "128270": {
-      "content": "<|reserved_special_token_264|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "128271": {
-      "content": "<|reserved_special_token_265|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "128272": {
-      "content": "<|reserved_special_token_266|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "128273": {
-      "content": "<|reserved_special_token_267|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "128274": {
-      "content": "<|reserved_special_token_268|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "128275": {
-      "content": "<|reserved_special_token_269|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "128276": {
-      "content": "<|reserved_special_token_270|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "128277": {
-      "content": "<|reserved_special_token_271|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "128278": {
-      "content": "<|reserved_special_token_272|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "128279": {
-      "content": "<|reserved_special_token_273|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "128280": {
-      "content": "<|reserved_special_token_274|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "128281": {
-      "content": "<|reserved_special_token_275|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "128282": {
-      "content": "<|reserved_special_token_276|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "128283": {
-      "content": "<|reserved_special_token_277|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "128284": {
-      "content": "<|reserved_special_token_278|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "128285": {
-      "content": "<|reserved_special_token_279|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "128286": {
-      "content": "<|reserved_special_token_280|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "128287": {
-      "content": "<|reserved_special_token_281|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
     }
   },
   "bos_token": "<|begin_of_text|>",
@@ -2313,7 +2057,7 @@
     "input_ids",
     "attention_mask"
   ],
-  "model_max_length": 8192,
+  "model_max_length": 32768,
   "pad_token": "<|end_of_text|>",
   "padding_side": "left",
   "tokenizer_class": "PreTrainedTokenizerFast"