Upload folder using huggingface_hub

.gitattributes

@@ -38,3 +38,10 @@ examples/caixukun.mp4 filter=lfs diff=lfs merge=lfs -text
 examples/flower.mp4 filter=lfs diff=lfs merge=lfs -text
 examples/food_menu.png filter=lfs diff=lfs merge=lfs -text
 examples/star_kun.mp4 filter=lfs diff=lfs merge=lfs -text
+model/lib/pointnet2/build/lib.linux-x86_64-cpython-39/pointnet2_cuda.cpython-39-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+model/lib/pointnet2/build/temp.linux-x86_64-cpython-39/src/ball_query.o filter=lfs diff=lfs merge=lfs -text
+model/lib/pointnet2/build/temp.linux-x86_64-cpython-39/src/group_points.o filter=lfs diff=lfs merge=lfs -text
+model/lib/pointnet2/build/temp.linux-x86_64-cpython-39/src/interpolate.o filter=lfs diff=lfs merge=lfs -text
+model/lib/pointnet2/build/temp.linux-x86_64-cpython-39/src/pointnet2_api.o filter=lfs diff=lfs merge=lfs -text
+model/lib/pointnet2/build/temp.linux-x86_64-cpython-39/src/sampling.o filter=lfs diff=lfs merge=lfs -text
+model/lib/pointnet2/dist/pointnet2-0.0.0-py3.9-linux-x86_64.egg filter=lfs diff=lfs merge=lfs -text

app.py

@@ -22,6 +22,10 @@ from data.fintune_dataset import make_audio_features
 from data import video_utils 
 from dataclasses import dataclass
 from huggingface_hub import hf_hub_download
+import plotly.graph_objects as go
+from data.fintune_dataset import pc_norm
+from functools import partial
+import glob
 
 T_random_resized_crop = transforms.Compose([
     transforms.RandomResizedCrop(size=(224, 224), scale=(0.9, 1.0), ratio=(0.75, 1.3333), interpolation=3,
@@ -39,6 +43,17 @@ def load_video(video_path):
     video_feats = video_utils.load_and_transform_video_data(video_path, video_path, clip_duration=1, clips_per_video=5)
     return video_feats[:, :, 0]
 
+def load_point(point_path):
+    point_feat = np.load(point_path)
+    point_feat = torch.tensor(point_feat)
+    point_feat = pc_norm(point_feat)
+    return point_feat
+
+def load_fmri(fmri_path):
+    data = np.load(fmri_path)
+    data = data.mean(axis=0)
+    data = torch.tensor(data[None])
+    return data
 
 def model_worker(
     rank: int, args: argparse.Namespace, barrier: mp.Barrier,
@@ -79,7 +94,8 @@ def model_worker(
     with default_tensor_type(dtype=target_dtype, device="cuda"):
         model = MetaModel(args.llama_type, args.llama_config, tokenizer_path=args.tokenizer_path)
     for ckpt_id in range(args.num_ckpts):
-        ckpt_path = hf_hub_download(repo_id=args.pretrained_path, filename=args.ckpt_format.format(str(ckpt_id)))
+        # ckpt_path = hf_hub_download(repo_id=args.pretrained_path, filename=args.ckpt_format.format(str(ckpt_id)))
+        ckpt_path = os.path.join(args.pretrained_path, args.ckpt_format.format(str(ckpt_id)))
         print(f"Loading pretrained weights {ckpt_path}")
         checkpoint = torch.load(ckpt_path, map_location='cpu')
         msg = model.load_state_dict(checkpoint, strict=False)
@@ -91,7 +107,7 @@ def model_worker(
     barrier.wait()
 
     while True:
-        img_path, audio_path, video_path, chatbot, max_gen_len, temperature, top_p, modality = request_queue.get()
+        img_path, audio_path, video_path, point_path, fmri_path, chatbot, max_gen_len, temperature, top_p, modality = request_queue.get()
         if 'image' in modality and img_path is not None:
             image = Image.open(img_path).convert('RGB')
             inputs = T_random_resized_crop(image)
@@ -99,6 +115,10 @@ def model_worker(
             inputs = load_video(video_path)
         elif 'audio' in modality and audio_path is not None:
             inputs = load_audio(audio_path)
+        elif 'point' in modality and point_path is not None:
+            inputs = load_point(point_path)
+        elif 'fmri' in modality and fmri_path is not None:
+            inputs = load_fmri(fmri_path)
         else:
             inputs = None
         
@@ -164,9 +184,9 @@ def gradio_worker(
     def show_user_input(msg, chatbot):
         return "", chatbot + [[msg, None]]
 
-    def stream_model_output(img_path, audio_path, video_path, chatbot, max_gen_len, gen_t, top_p, modality):
+    def stream_model_output(img_path, audio_path, video_path, point_path, fmri_path, chatbot, max_gen_len, gen_t, top_p, modality):
         for queue in request_queues:
-            queue.put((img_path, audio_path, video_path, chatbot, max_gen_len, gen_t, top_p, modality))
+            queue.put((img_path, audio_path, video_path, point_path, fmri_path, chatbot, max_gen_len, gen_t, top_p, modality))
         while True:
             content_piece = response_queue.get()
             chatbot[-1][1] = content_piece["text"]
@@ -184,12 +204,27 @@ def gradio_worker(
         msg = ""
         return chatbot, msg
     
-    def change_modality_image():
-        return 'image'
-    def change_modality_video():
-        return 'video'
-    def change_modality_audio():
-        return 'audio'
+    def show_point_cloud(file):
+        point = load_point(file).numpy()
+        fig = go.Figure(
+            data=[
+                go.Scatter3d(
+                    x=point[:,0], y=point[:,1], z=point[:,2],
+                    mode='markers',
+                    marker=dict(
+                    size=1.2,
+                    color=['rgb({},{},{})'.format(r, g, b) for r,g,b in zip(point[:,3], point[:,4], point[:,5])]
+                ))],
+            layout=dict(
+                scene=dict(
+                    xaxis=dict(visible=False),
+                    yaxis=dict(visible=False),
+                    zaxis=dict(visible=False)
+                )),)
+        return fig
+    
+    def change_modality(modal):
+        return modal
 
     CSS ="""
     .contain { display: flex; flex-direction: column; }
@@ -235,11 +270,28 @@ def gradio_worker(
                         inputs=[audio_path],
                     )
                 with gr.Tab('Point Cloud') as point_tab:
-                    gr.Markdown('Coming soon🤗')
+                    point_path = gr.File(label='Point Cloud Input', elem_id="pointpath", elem_classes="")
+                    point_vis = gr.Plot()
+                    btn = gr.Button(value="Show Point Cloud")
+                    btn.click(show_point_cloud, point_path, point_vis)
+                    gr.Examples(
+                        examples=glob.glob("examples/point/*.npy"),
+                        inputs=[point_path],
+                        examples_per_page=5,
+                    )
                 with gr.Tab('IMU') as imu_tab:
                     gr.Markdown('Coming soon🤗')
                 with gr.Tab('fMRI') as fmri_tab:
-                    gr.Markdown('Coming soon🤗')
+                    fmri_path = gr.File(label='fMRI Input', elem_id="fmripath", elem_classes="")
+                    fmri_image_path = gr.Image(interactive=False)
+                    gr.Examples(
+                        examples=[
+                            [file.replace('.jpg', '.npy'), file]
+                            for file in glob.glob("examples/fmri/*.jpg")
+                        ],
+                        inputs=[fmri_path, fmri_image_path],
+                        examples_per_page=3,
+                    )
                 with gr.Tab('Depth Map') as depth_tab:
                     gr.Markdown('Coming soon🤗')
                 with gr.Tab('Normal Map') as normal_tab:
@@ -252,7 +304,7 @@ def gradio_worker(
                 with gr.Row():
                     submit_button = gr.Button("Submit", variant="primary")
                     undo_button = gr.Button("Undo")
-                    clear_button = gr.ClearButton([chatbot, msg, img_path, audio_path, video_path])
+                    clear_button = gr.ClearButton([chatbot, msg, img_path, audio_path, video_path, point_path, fmri_path, point_vis])
                 with gr.Row():
                     max_gen_len = gr.Slider(
                         minimum=1, maximum=args.model_max_seq_len // 2,
@@ -268,19 +320,21 @@ def gradio_worker(
                         label="Top-p",
                     )
         
-        img_tab.select(change_modality_image, [], [modality])
-        video_tab.select(change_modality_video, [], [modality])
-        audio_tab.select(change_modality_audio, [], [modality])
+        img_tab.select(partial(change_modality, 'image'), [], [modality])
+        video_tab.select(partial(change_modality, 'video'), [], [modality])
+        audio_tab.select(partial(change_modality, 'audio'), [], [modality])
+        point_tab.select(partial(change_modality, 'point'), [], [modality])
+        fmri_tab.select(partial(change_modality, 'fmri'), [], [modality])
 
         msg.submit(
             show_user_input, [msg, chatbot], [msg, chatbot],
         ).then(
-            stream_model_output, [img_path, audio_path, video_path, chatbot, max_gen_len, gen_t, top_p, modality], chatbot,
+            stream_model_output, [img_path, audio_path, video_path, point_path, fmri_path, chatbot, max_gen_len, gen_t, top_p, modality], chatbot,
         )
         submit_button.click(
             show_user_input, [msg, chatbot], [msg, chatbot],
         ).then(
-            stream_model_output, [img_path, audio_path, video_path, chatbot, max_gen_len, gen_t, top_p, modality], chatbot,
+            stream_model_output, [img_path, audio_path, video_path, point_path, fmri_path, chatbot, max_gen_len, gen_t, top_p, modality], chatbot,
         )
         undo_button.click(undo, chatbot, chatbot)
         # img_path.change(clear, [], [chatbot, msg])
@@ -297,10 +351,11 @@ class DemoConfig:
     model_max_seq_len = 2048
     # pretrained_path = "weights/7B_2048/consolidated.00-of-01.pth"
     # pretrained_path = hf_hub_download(repo_id="csuhan/OneLLM-7B", filename="consolidated.00-of-01.pth")
-    pretrained_path = "csuhan/OneLLM-7B-hf"
+    # pretrained_path = "csuhan/OneLLM-7B-hf"
+    pretrained_path = "/home/pgao/jiaming/weights/7B_v20_splits/"
     ckpt_format = "consolidated.00-of-01.s{}.pth"
     num_ckpts = 10
-    master_port = 23861
+    master_port = 23863
     master_addr = "127.0.0.1"
     dtype = "fp16"
 

examples/fmri/sample000000498.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:db81853dbaf9ed1869162d9f7218e6b53edf6ea8fac391ef181f59304c3918d4
+size 94472

examples/fmri/sample000000789.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e659eefe264f472847e496b4990459300fbb73587f0f99f69f630b3bdcd86b53
+size 94472

examples/fmri/sample000001037.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e62dcb5ad478c8d50e3b60d558e4df56a03c1f3999883da2719c43464db48dc4
+size 94472

examples/fmri/sample000001554.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:54e647f3823c878647b2ef18966e033b6f9d865016afc30be2cd06e09b3511f8
+size 94472

examples/fmri/sample000002995.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a9d9187274787d4f1d07292c9c25ea117f7810902b75cd9434aade4a7d659f9d
+size 94472

examples/fmri/sample000004364.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1e4800075aa129fd3bbdd27c63c7597b375cac3552555ac1173db11542275125
+size 94472

examples/fmri/sample000004550.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:64f6c75ff93ddb5988fc07f861fe4ff053996dcd7d72a96807fce77efbb82cd7
+size 94472

examples/fmri/sample000005047.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8e44a51e7f0875a3a5d9b132ccc9f2f0ce2af13a566956272773727f80f2afa3
+size 94472

examples/fmri/sample000005139.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7d08f633cc921f084de2512717b08f14631581c74eacfaf7e2e025b59adc377b
+size 94472

examples/point/0031ba19d3e042c4bcf79eba40ccc812_8192.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d3d3c8cd5e953e30bc86a706e1c69d629bde6711820e3ba4d106ccd1befe7686
+size 196736

examples/point/1a282b70b3f14fd79d739891e4327df5_8192.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:874bc35c0b23a5bc6063e71b420ae51b82190862b87b4c37e8295f5ca4be2e05
+size 196736

examples/point/1ee7fc4081ac47ccaae8112b6e925c51_8192.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:31d011e25b3745bcd9f3266b9d9f4b4cd03d680f007b6fc46de7b621dbc2bbb4
+size 196736

examples/point/2221aec5809d43d88aedf82738432318_8192.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d7569c06b11aaea351477d244d8e76152501190f16e28227de2ebf7ba5354c0b
+size 196736

examples/point/3061e9a9236d4b98b74c95ff189d28a0_8192.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3202eed35e0259d8ff00a49c35b2f0f61324fd90da78ea832f5c356fccd9ea35
+size 196736

examples/point/39930c4d685f4a9c8f7bd63c1a2a7dee_8192.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4eafe3bed10c54cf0f862982a36bf4edf5cb7fccb8e0af7c0188a5a900d61286
+size 196736

examples/point/3a61b9d2451c4df2a1c6a6550e1875df_8192.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6f9be8648887b54273884df915f1f351877499fb6092f2e85b6cfaf67b03476e
+size 196736

examples/point/442e0532a7634ad1ba22d28534267e2f_8192.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cf7dfbf43f9794c8a5c90033eb458010d13defa6dc04a1d61b274675e248bfa7
+size 196736

examples/point/463fe91df66e44a681ebbcd7f7ba59e4_8192.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:21033208a5418f923e1c281d5bdd81ac75b09c6d047030350a718bd50d8f5185
+size 196736

examples/point/494854c6ffd540339a1bc6bd91c0349b_8192.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c5a87443e5b14abcf04738ae9ceb01c62e87c3b5108e1db147880b7bdb700160
+size 196736

examples/point/4adbacc99de44b5597ce3e1f62026fc1_8192.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:81d79afe15fa2c011dd82c7cfbbe17bb062b364824abcb37f7b3da68a763a4c3
+size 196736

examples/point/4d6f0a6aab8e45f4af131ac517c74363_8192.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ad1e243593b19e7e58a77f0512a8f892d663142853efac1e56c5a83db78cc50
+size 196736

examples/point/541747a461f84142a113e2f591a7eb9b_8192.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d00c235c92766c9dde832209df18f036e68c74b11e477f40d283cde1af8b1ea2
+size 196736

examples/point/6796698b4d564377838c06319ec96f90_8192.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:da91660e20b77a3f9f8a8ab173cc6c0998063282d387049906252bbf661f0b5f
+size 196736

examples/point/69865c89fc7344be8ed5c1a54dbddc20_8192.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:99a8363e432756f90425b6a5192cf5b8a547a5bd009577e72203fd42fd2319cf
+size 196736

examples/point/7610a160fbaf41d681c7d7b9627df85d_8192.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca43630b2f139a5b0a90633f1d13d3f61e4432bf9976de5c233791ab57553dae
+size 196736

examples/point/7f309e755a0c4b69b64e667977268087_8192.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0add1f7bb1ad3f4b70c383431e7a2ac372c8e4377d2987c4fc08f6bd6776d5af
+size 196736

examples/point/844518dcccf44c86ad1ead660ab103fb_8192.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:60c2a1284188bf60e405e27f2008030247657ea256e9e3f95940afde294a8080
+size 196736

examples/point/88867268d228490da73b99d3126dc25c_8192.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:53bca2ad577b7eaf2c531379f69d9d904ad632a57205763ff5e17f2d2bfa7716
+size 196736

examples/point/8c833864d0bb40f4b3f3a3fe1e99e702_8192.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:89dff97c8370e615e5674b17a7ef527b24af2bfbfe764294a5efa103064ed887
+size 196736

examples/point/983fa8b23a084f5dacd157e6c9ceba97_8192.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b7a67c55f4bc9a29edb9eaf458bfdcc67cb93fad504da69f6afe0e10cf1c1e1e
+size 196736

examples/point/9e9fa9bb1761412bb0e7ca9913f64e91_8192.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:afcffb80b428ee8728a9d30b0c7643655a1419a41872f4313082ad6768ee1eb2
+size 196736

examples/point/a4a0a21eedae4d82b0deac43b61dc4a3_8192.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:895b30c7ac4e89e53b4beaac9199c33d1bd4fa8a39f15fa52fe237365bad9ed4
+size 196736

examples/point/abc413b5e2d14a028dab615b49777167_8192.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:17dd6a38a6a0ad22e1ac17265d49b50083f196cbd8bfeb1f67cdfbc8df5eacc8
+size 196736

examples/point/b65b0ad074664b4aa624583e52fa685e_8192.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:06e1b52c90abd98dc01adfa51048dc16806abbd4609c8cb1ab12bb99e52f65d3
+size 196736

examples/point/c20eb3a5a93e4cddb06c2f98626b1830_8192.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3793341c5f418f53b09b13e50a969b9bbb6ac865c4c59c273bbf41a06f1bc238
+size 196736

examples/point/d43136ef2b1a46998b7214df1a15b72c_8192.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:32cd2cd274d12c93873f41653a3d94599a8ab1108df029365f20cfd474f6bf38
+size 196736

examples/point/d4aa4126fce64397935d128ed54c3def_8192.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2cc8f49dcfeb8aca490d2ac7f08e6790ca09ed464f4be8b744710d365c23a487
+size 196736

examples/point/dd491278516040998667f9dd998fad4f_8192.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1e4d509ddcd27c36f7c413cc31b5372cc4510cf8d73df0521107b404279c4698
+size 196736