Hugging Face's logo

paulml
/
MoMoAlpaca-72b

Text Generation
Transformers
Safetensors
llama
Merge
mergekit
lazymergekit
ibivibiv/alpaca-dragon-72b-v1
moreh/MoMo-72B-lora-1.8.7-DPO
text-generation-inference
Inference Endpoints
Model card Files Community
paulml commited on
Commit
16c2ae2
•
1 Parent(s): 81efe09

Upload folder using huggingface_hub

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. README.md +63 -0
  2. config.json +30 -0
  3. mergekit_config.yml +17 -0
  4. merges.txt +0 -0
  5. model-00001-of-00082.safetensors +3 -0
  6. model-00002-of-00082.safetensors +3 -0
  7. model-00003-of-00082.safetensors +3 -0
  8. model-00004-of-00082.safetensors +3 -0
  9. model-00005-of-00082.safetensors +3 -0
  10. model-00006-of-00082.safetensors +3 -0
  11. model-00007-of-00082.safetensors +3 -0
  12. model-00008-of-00082.safetensors +3 -0
  13. model-00009-of-00082.safetensors +3 -0
  14. model-00010-of-00082.safetensors +3 -0
  15. model-00011-of-00082.safetensors +3 -0
  16. model-00012-of-00082.safetensors +3 -0
  17. model-00013-of-00082.safetensors +3 -0
  18. model-00014-of-00082.safetensors +3 -0
  19. model-00015-of-00082.safetensors +3 -0
  20. model-00016-of-00082.safetensors +3 -0
  21. model-00017-of-00082.safetensors +3 -0
  22. model-00018-of-00082.safetensors +3 -0
  23. model-00019-of-00082.safetensors +3 -0
  24. model-00020-of-00082.safetensors +3 -0
  25. model-00021-of-00082.safetensors +3 -0
  26. model-00022-of-00082.safetensors +3 -0
  27. model-00023-of-00082.safetensors +3 -0
  28. model-00024-of-00082.safetensors +3 -0
  29. model-00025-of-00082.safetensors +3 -0
  30. model-00026-of-00082.safetensors +3 -0
  31. model-00027-of-00082.safetensors +3 -0
  32. model-00028-of-00082.safetensors +3 -0
  33. model-00029-of-00082.safetensors +3 -0
  34. model-00030-of-00082.safetensors +3 -0
  35. model-00031-of-00082.safetensors +3 -0
  36. model-00032-of-00082.safetensors +3 -0
  37. model-00033-of-00082.safetensors +3 -0
  38. model-00034-of-00082.safetensors +3 -0
  39. model-00035-of-00082.safetensors +3 -0
  40. model-00036-of-00082.safetensors +3 -0
  41. model-00037-of-00082.safetensors +3 -0
  42. model-00038-of-00082.safetensors +3 -0
  43. model-00039-of-00082.safetensors +3 -0
  44. model-00040-of-00082.safetensors +3 -0
  45. model-00041-of-00082.safetensors +3 -0
  46. model-00042-of-00082.safetensors +3 -0
  47. model-00043-of-00082.safetensors +3 -0
  48. model-00044-of-00082.safetensors +3 -0
  49. model-00045-of-00082.safetensors +3 -0
  50. model-00046-of-00082.safetensors +3 -0
README.md ADDED
@@ -0,0 +1,63 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ tags:
3
+ - merge
4
+ - mergekit
5
+ - lazymergekit
6
+ - ibivibiv/alpaca-dragon-72b-v1
7
+ - moreh/MoMo-72B-lora-1.8.7-DPO
8
+ base_model:
9
+ - ibivibiv/alpaca-dragon-72b-v1
10
+ - moreh/MoMo-72B-lora-1.8.7-DPO
11
+ ---
12
+
13
+ # MoMoAlpaca-72b
14
+
15
+ MoMoAlpaca-72b is a merge of the following models using [LazyMergekit](https://colab.research.google.com/drive/1obulZ1ROXHjYLn6PPZJwRR6GzgQogxxb?usp=sharing):
16
+ * [ibivibiv/alpaca-dragon-72b-v1](https://huggingface.co/ibivibiv/alpaca-dragon-72b-v1)
17
+ * [moreh/MoMo-72B-lora-1.8.7-DPO](https://huggingface.co/moreh/MoMo-72B-lora-1.8.7-DPO)
18
+
19
+ ## 🧩 Configuration
20
+
21
+ ```yaml
22
+ slices:
23
+ - sources:
24
+ - model: ibivibiv/alpaca-dragon-72b-v1
25
+ layer_range: [0, 80]
26
+ - model: moreh/MoMo-72B-lora-1.8.7-DPO
27
+ layer_range: [0, 80]
28
+ merge_method: slerp
29
+ base_model: ibivibiv/alpaca-dragon-72b-v1
30
+ parameters:
31
+ t:
32
+ - filter: self_attn
33
+ value: [0, 0.5, 0.3, 0.7, 1]
34
+ - filter: mlp
35
+ value: [1, 0.5, 0.7, 0.3, 0]
36
+ - value: 0.5
37
+ dtype: float32
38
+ ```
39
+
40
+ ## 💻 Usage
41
+
42
+ ```python
43
+ !pip install -qU transformers accelerate
44
+
45
+ from transformers import AutoTokenizer
46
+ import transformers
47
+ import torch
48
+
49
+ model = "paulml/MoMoAlpaca-72b"
50
+ messages = [{"role": "user", "content": "What is a large language model?"}]
51
+
52
+ tokenizer = AutoTokenizer.from_pretrained(model)
53
+ prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
54
+ pipeline = transformers.pipeline(
55
+ "text-generation",
56
+ model=model,
57
+ torch_dtype=torch.float16,
58
+ device_map="auto",
59
+ )
60
+
61
+ outputs = pipeline(prompt, max_new_tokens=256, do_sample=True, temperature=0.7, top_k=50, top_p=0.95)
62
+ print(outputs[0]["generated_text"])
63
+ ```
config.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "ibivibiv/alpaca-dragon-72b-v1",
3
+ "architectures": [
4
+ "LlamaForCausalLM"
5
+ ],
6
+ "attention_bias": true,
7
+ "attention_dropout": 0.0,
8
+ "bos_token_id": 1,
9
+ "eos_token_id": 151643,
10
+ "hidden_act": "silu",
11
+ "hidden_size": 8192,
12
+ "initializer_range": 0.02,
13
+ "intermediate_size": 24576,
14
+ "max_position_embeddings": 32768,
15
+ "model_type": "llama",
16
+ "num_attention_heads": 64,
17
+ "num_hidden_layers": 80,
18
+ "num_key_value_heads": 64,
19
+ "pad_token_id": 151643,
20
+ "pretraining_tp": 1,
21
+ "rms_norm_eps": 1e-06,
22
+ "rope_scaling": null,
23
+ "rope_theta": 1000000,
24
+ "seq_length": 32768,
25
+ "tie_word_embeddings": false,
26
+ "torch_dtype": "float32",
27
+ "transformers_version": "4.37.2",
28
+ "use_cache": true,
29
+ "vocab_size": 152064
30
+ }
mergekit_config.yml ADDED
@@ -0,0 +1,17 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+ slices:
3
+ - sources:
4
+ - model: ibivibiv/alpaca-dragon-72b-v1
5
+ layer_range: [0, 80]
6
+ - model: moreh/MoMo-72B-lora-1.8.7-DPO
7
+ layer_range: [0, 80]
8
+ merge_method: slerp
9
+ base_model: ibivibiv/alpaca-dragon-72b-v1
10
+ parameters:
11
+ t:
12
+ - filter: self_attn
13
+ value: [0, 0.5, 0.3, 0.7, 1]
14
+ - filter: mlp
15
+ value: [1, 0.5, 0.7, 0.3, 0]
16
+ - value: 0.5
17
+ dtype: float32
merges.txt ADDED
The diff for this file is too large to render. See raw diff
 
model-00001-of-00082.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:93c877a3c02521f0fbff6812c0163a07aa9e3f24faaf2e7a0f4a3c6a78c88670
3
+ size 4982833288
model-00002-of-00082.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:337d4b73cacc7d659934f665901d6ffd0928aeb3acbecfc9704ef53afb1eb78b
3
+ size 3758130224
model-00003-of-00082.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:773415ee34144364a8d44d03879fb5a90d62b6b8e5aa04cc753c7aa164a8d6da
3
+ size 3489694864
model-00004-of-00082.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2b048257df29ac97962449f4d0bed1e67f1a12299fe3b4001704199635272567
3
+ size 3758162856
model-00005-of-00082.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a0b4c6cd764de4bcdacb0ca62212b62f21e4cb55aafec1103c6e3fe994e735ae
3
+ size 3489727520
model-00006-of-00082.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bc9da201200765f7ddc74885c7a4e5a7cbc35621f085ce09e48c0d8ee23af233
3
+ size 3489793280
model-00007-of-00082.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1ac436de3aba7ffb14c6c2955037f45cfc056030425274317eddf8454606aef6
3
+ size 3489661768
model-00008-of-00082.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:119e3d1b510be84d717885c4bc15ccde66d93b93395547c29246e92398b46245
3
+ size 3758163088
model-00009-of-00082.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5fdba2521dfff73b22901b6d51652625e75f07403b87719ba1642f61394078b1
3
+ size 3221291944
model-00010-of-00082.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:302a91d7d6a63b2d927f9dcc289ea04a659e75811f5cb524a99dfb13532ad97f
3
+ size 3489793280
model-00011-of-00082.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:76cc0321cb892fa8d3dd462ef5e39046a72f9c15b31aa02fba706a8fd916b6ec
3
+ size 3489661768
model-00012-of-00082.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c8e10ec71a558c32931cc4f450924de7264708ef5a5d49b1de85d002a850bb5c
3
+ size 3758163104
model-00013-of-00082.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a8cbf5db645282817d526dcf7fe196fd4ceb74f0eab1895a1c9e1deeee80b8ec
3
+ size 3221291952
model-00014-of-00082.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a6eef3d8049ba337ed903b8245a14c08fa4ae79a3541e959875f84e2b2c15a11
3
+ size 3489793288
model-00015-of-00082.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:67c225276d0f260d0cb1e53d298d0dc3a56825ef8c18b87c4de88714e460f869
3
+ size 3489661776
model-00016-of-00082.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:01d24923736b2c3e7501d7281a415bcc842e92716d91761ab80d93b56ad9d560
3
+ size 3758163096
model-00017-of-00082.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6f8d3110f54750fe9f1748a3ec162805d2dd9ae55df54f0b6993e6bb690f6ff6
3
+ size 3221291952
model-00018-of-00082.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a3a55b4f7abe040c899e7b8c09f3122a12268c450fd2353bad56edde9a5d791e
3
+ size 3489793288
model-00019-of-00082.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:50a2496b3c5524a7f40267171d6d420e63ba11637e0bf721d17703acbbfbca7e
3
+ size 3489661776
model-00020-of-00082.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4503180398322e9acdc14ec44f57544cede96a9037c8aedb52e6a021bad85be5
3
+ size 3758163096
model-00021-of-00082.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:534b3aa0d912123551d1892e18bb960ec7cc0bdb481c8631cb08c3311dad88e6
3
+ size 3221291952
model-00022-of-00082.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:93d032f6d04ef3598f1593cb827daef9a1d1d70207a20a1a661895edd0a54940
3
+ size 3489793288
model-00023-of-00082.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3e6157961361b0af60bf30c7175d0b6b2ab57473c4eaafe2ec711326f8482dc9
3
+ size 3489661776
model-00024-of-00082.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e73cd8a997ddab19f9933a7d6edf2ef7b1272ade7afb10ce0ec31062c00d0f81
3
+ size 3758163096
model-00025-of-00082.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:25e262a3241c66ddeb52a3fe5c2f9eb89c7ba85c875304d6fff3b6db6beef8d2
3
+ size 3221291952
model-00026-of-00082.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f22c2a8cc3aa8b9e7f7e0919d9f207ce56c209ea151dfa50c88d29ce6780181f
3
+ size 3489793288
model-00027-of-00082.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ee438d6935b4b08cb1a285bea52e8cb3753817de21b3e8360859674e75c96b63
3
+ size 3489661776
model-00028-of-00082.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c62b6027a7951f4ebb192be773d2d516a39bc81c9a1d772649942c231e529621
3
+ size 3758163096
model-00029-of-00082.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:dbf88ad613fcdcaaadca747a9dd438c3e13fdf566d8a650a9438fea9616d6186
3
+ size 3221291952
model-00030-of-00082.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d7e486153ad0f05b46f8fc89dd6f7e44e36c64d09e285e3f12d13cb45b821685
3
+ size 3489793288
model-00031-of-00082.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:da8e6ff7a6e79eb9550f0026eaf2b89cdb1126be9953ee96fa715ff96c366907
3
+ size 3489661776
model-00032-of-00082.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:77fe566037dad0f7132f80b17e7f7821298523739717e3cc13a22922dd6d18c5
3
+ size 3758163096
model-00033-of-00082.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ab54352cfaddb96412abfae57957e7e4c967d5efc944ce1ff082b28fe37dd8b4
3
+ size 3221291952
model-00034-of-00082.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a23e87f843df6e6099ce499802d30d75edd9a0f555b526ff52182cce49eb53cb
3
+ size 3489793288
model-00035-of-00082.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f3ec5b460b832abf8fe6c2874b2e2d1b80acc58c94f60b156e28e386a43fea8f
3
+ size 3489661776
model-00036-of-00082.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6898e3e1c937dfa91b0dbca954598b287f9ae3869b5ef30d3a6245dc139dc047
3
+ size 3758163096
model-00037-of-00082.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:00513cb364ceaa1a235ae68b79951da06950c8cf76216056f8bb47e216e54570
3
+ size 3221291952
model-00038-of-00082.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0eac04914885c4210bcc6738e04d533d7d2196c5ad1ad69c6175045545702a6c
3
+ size 3489793288
model-00039-of-00082.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b867d71213f2d28dcfbff41f51596aeca94db8cb5f61010e6901c4ff29f0df3e
3
+ size 3489661776
model-00040-of-00082.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a6db3936be8cc89e9c41da665e52969218a5b548345856e87fee2c55cb67ab21
3
+ size 3758163096
model-00041-of-00082.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2d3fc3e8f787db899edc58b39c805a795f616f6ad2ee977acb4e05de1c1fe55c
3
+ size 3221291952
model-00042-of-00082.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a29e9e5ab77278feed424878453c993f6662f0d81a98a10d8c66cfd0b98567a9
3
+ size 3489793288
model-00043-of-00082.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c6b71a8806765ff7ffe187c010140b8dcae58d200d777f21997cd2a98d291a57
3
+ size 3489661776
model-00044-of-00082.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:49825ae2f3951cd8ac9037465d03ac15cf7b823a4d70c697fa3233a61aa5b9c5
3
+ size 3758163096
model-00045-of-00082.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d26bf9488dabfb8c59ab1cad1a8e66bb035360ccea259890d4a5aef0fe6d2eb5
3
+ size 3221291952
model-00046-of-00082.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:43c79d48a9cb0c1a664c12d0ec27253bed7f87500a9ea594050cde287a42c182
3
+ size 3489793288