Upload 12 files

Nebula-baseline/NeBuLa-org_adapter_config.json

@@ -0,0 +1,26 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "/tmpdir/chaturve/llama-3-8b-hf/",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 16,
+  "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 64,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "q_proj",
+    "v_proj"
+  ],
+  "task_type": "CAUSAL_LM"
+}

Nebula-baseline/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff3d5d873f8094e23565ca85b675aca9b86457c03bafb339967ffd49eabe50c0
+size 109069176

Nebula-synth/NeBuLa-synth_adapter_config.json

@@ -0,0 +1,26 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "/tmpdir/chaturve/llama-3-8b-hf/",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 16,
+  "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 64,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "q_proj",
+    "v_proj"
+  ],
+  "task_type": "CAUSAL_LM"
+}

Nebula-synth/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ead40404c9439f63060938a58d0f406a56977775099a6723af2e41e36247dd8e
+size 109069176

README.md

@@ -0,0 +1,25 @@
+---
+pipeline_tag: other
+tags:
+- minecraft
+- action prediction
+language:
+- en
+---
+
+# Nebula: A Minecraft Neural Builder
+
+Nebula is an LLM (Llama3-8B) finetuned on the minecraft action prediction task (https://aclanthology.org/2020.acl-main.232/). 
+
+## Model Details
+We provide two variants of the model: Nebula-baseline, which was trained on minecraft data set; and Nebula-synth, which was trained on level-1
+and level-2 synthetic data along with the minecraft data set. Both the models achieve similar net-action F1 score on minecraft test set.
+
+### Model Description
+
+- **Language(s) (NLP):** English
+- **Finetuned from model:** Llama3-8B
+
+### Model Sources
+
+**Paper:** https://arxiv.org/abs/2406.18164

merge.py

@@ -0,0 +1,56 @@
+import torch
+import json
+from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
+from datasets import load_dataset
+from peft import LoraConfig, PeftModel
+
+device_map = "auto"
+model = AutoModelForCausalLM.from_pretrained(
+    "/path/to/meta-llama3-8b",
+    #low_cpu_mem_usage=True,
+    return_dict=True,
+    torch_dtype=torch.float16,
+    device_map=device_map,
+)
+
+model = PeftModel.from_pretrained(model, "/path/to/llama3-8b-adapter", device_map=device_map)
+model = model.merge_and_unload()
+
+tokenizer = AutoTokenizer.from_pretrained("/path/to/meta-llama3-8b", trust_remote_code=True)
+tokenizer.pad_token_id = tokenizer.eos_token_id + 1
+model.config.pad_token_id = tokenizer.pad_token_id
+pipe = pipeline(task="text-generation", model=model, tokenizer=tokenizer, max_length=4096, do_sample=False)
+print("Padding side:",tokenizer.padding_side)
+val_dataset = load_dataset("csv", data_files={'val':'/path/to/actseq-val-new.csv'})["val"]
+test_dataset = load_dataset("csv", data_files={'test':'/path/to/actseq-test-new.csv'})["test"]
+
+
+def formatting_prompts_func(example):
+     output_texts = []
+     for i in range(len(example['dial_with_actions'])):
+         text = f"<|begin_of_text|>Predict the action sequence (AS) for the Minecraft excerpt:\n {example['dial_with_actions'][i]}\n ### AS:"
+         output_texts.append(text)
+     return output_texts
+
+
+val_texts = formatting_prompts_func(val_dataset)
+test_texts = formatting_prompts_func(test_dataset)
+
+print("Val Length:", len(val_texts))
+print("Test Length:", len(test_texts))
+
+f = open("/path/to/val-output-file","w")
+
+for text in val_texts:
+    print(text)
+    print(pipe(text)[0]["generated_text"], file=f)
+
+f.close()
+
+f = open("/path/to/test-output-file","w")
+
+for text in test_texts:
+    print(text)
+    print(pipe(text)[0]["generated_text"], file=f)
+
+f.close()

requirements.txt

@@ -0,0 +1,90 @@
+accelerate==0.27.2
+aiohttp==3.9.3
+aiosignal==1.3.1
+async-timeout==4.0.3
+attrs==23.2.0
+bitsandbytes==0.41.3
+brotlipy==0.7.0
+certifi==2020.6.20
+cffi==1.14.3
+chardet==3.0.4
+conda-package-handling==1.7.2
+cryptography==3.2.1
+datasets==2.17.1
+dill==0.3.8
+docstring-parser==0.15
+filelock==3.13.1
+frozenlist==1.4.1
+fsspec==2023.10.0
+huggingface-hub==0.20.3
+idna==2.10
+Jinja2==3.1.3
+markdown-it-py==3.0.0
+MarkupSafe==2.1.5
+mdurl==0.1.2
+mpmath==1.3.0
+multidict==6.0.5
+multiprocess==0.70.16
+networkx==3.1
+numpy==1.24.4
+nvidia-cublas-cu11==11.11.3.6
+nvidia-cublas-cu12==12.1.3.1
+nvidia-cuda-cupti-cu11==11.8.87
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cuda-nvrtc-cu11==11.8.89
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-runtime-cu11==11.8.89
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cudnn-cu11==8.7.0.84
+nvidia-cudnn-cu12==8.9.2.26
+nvidia-cufft-cu11==10.9.0.58
+nvidia-cufft-cu12==11.0.2.54
+nvidia-curand-cu11==10.3.0.86
+nvidia-curand-cu12==10.3.2.106
+nvidia-cusolver-cu11==11.4.1.48
+nvidia-cusolver-cu12==11.4.5.107
+nvidia-cusparse-cu11==11.7.5.86
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-nccl-cu11==2.19.3
+nvidia-nccl-cu12==2.19.3
+nvidia-nvjitlink-cu12==12.3.101
+nvidia-nvtx-cu11==11.8.86
+nvidia-nvtx-cu12==12.1.105
+packaging==23.2
+pandas==2.0.3
+peft==0.7.1
+pillow==10.2.0
+pip==20.2.4
+psutil==5.9.8
+pyarrow==15.0.0
+pyarrow-hotfix==0.6
+pycosat==0.6.3
+pycparser==2.20
+pygments==2.17.2
+pyOpenSSL==19.1.0
+PySocks==1.7.1
+python-dateutil==2.8.2
+pytz==2024.1
+PyYAML==6.0.1
+regex==2023.12.25
+requests==2.24.0
+rich==13.7.0
+ruamel-yaml==0.15.87
+safetensors==0.4.2
+scipy==1.10.1
+setuptools==50.3.1.post20201107
+shtab==1.7.0
+six==1.15.0
+sympy==1.12
+tokenizers==0.15.2
+tqdm==4.66.2
+transformers==4.38.1
+triton==2.2.0
+trl==0.7.11
+typing-extensions==4.9.0
+tyro==0.7.3
+tzdata==2024.1
+urllib3==1.25.11
+wheel==0.35.1
+xxhash==3.4.1
+yarl==1.9.4