600M TIME

app.py

@@ -8,7 +8,7 @@ demo = gr.Interface(fn=gpt.get_response, inputs=["textbox",
                                                 gr.Slider(0, 100, value=50, step=1),
                                                 gr.Slider(0.1, 2.0, value=1.0),
                                                 gr.Dropdown(
-            ["mike-chat", "mike-code"], value="mike-chat"),
+            ["mike-chat", "mike-code", "mike-code-600m"], value="mike-chat"),
                                                  ], outputs=gr.Markdown(line_breaks=True), title="Mike Chat", article="""Mike is the greatest AI ever created. It was trained for about 8 hrs on my pc using fineweb-edu and open orca datasets. While it hallucinates a lot, it seems to be about on par with other lms of its size (about 160M params). Model details:
                                                  block_size: 512
                                                  n_layers: 12

gpt.py

@@ -134,16 +134,23 @@ my_GPT = load_compiled_model_state_dict(my_GPT, 'latest_model_finetune.pth')
 #my_GPT.load_state_dict(torch.load('latest_model_finetune.pth', map_location=torch.device('cpu')))
 my_GPT.eval()
 
-my_GPT_code = GPT(enc.n_vocab, 256, n_layers, n_heads, d_model, dropout=0.1) #enc.n_vocab
+my_GPT_code = GPT(enc.n_vocab, 256, n_layers, n_heads, d_model, dropout=0.0) #enc.n_vocab
 my_GPT_code = my_GPT_code.to(device)
 #my_GPT = torch.compile(my_GPT, mode='reduce-overhead')
 my_GPT_code = load_compiled_model_state_dict(my_GPT_code, 'mike-code-15k.pth')
 #my_GPT.load_state_dict(torch.load('latest_model_finetune.pth', map_location=torch.device('cpu')))
 my_GPT_code.eval()
 
+my_GPT_code_600 = GPT(enc.n_vocab, 256, 16, n_heads, 768 * 2, dropout=0.0) #enc.n_vocab
+my_GPT_code_600 = my_GPT_code_600.to(device)
+#my_GPT = torch.compile(my_GPT, mode='reduce-overhead')
+my_GPT_code_600 = load_compiled_model_state_dict(my_GPT_code_600, 'mike-code-600m.pth')
+#my_GPT.load_state_dict(torch.load('latest_model_finetune.pth', map_location=torch.device('cpu')))
+my_GPT_code_600.eval()
 models = {
    "mike-chat": my_GPT,
-   "mike-code": my_GPT_code
+   "mike-code": my_GPT_code,
+   "mike-code-600m": my_GPT_code_600
 }
 
 eot = enc._special_tokens['<|endoftext|>']

mike-code-600m.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:721dc760156cba2bfcfe6556e6867b7c0f8c03fc3d307cbccd78d2934544e440
+size 2432463763