add image projection weights and configs

app.py

@@ -1,5 +1,5 @@
-from models import PoemTextModel
-from inference import predict_poems_from_text
+from models import CLIPModel
+from inference import predict_poems_from_image
 from utils import get_poem_embeddings
 import config as CFG
 import json
@@ -10,7 +10,11 @@ def greet_user(name):
 	return "Hello " + name + " Welcome to Gradio!😎"
 
 if __name__ == "__main__":
-    model = PoemTextModel(poem_encoder_pretrained=True, text_encoder_pretrained=True).to(CFG.device)
+    model = CLIPModel(image_encoder_pretrained=True, 
+                text_encoder_pretrained=True, 
+                text_projection_trainable=False,
+                is_image_poem_pair=True
+                ).to(CFG.device)
     model.eval()
     # Inference: Output some example predictions and write them in a file
     with open('poem_embeddings.json', encoding="utf-8") as f:
@@ -20,17 +24,14 @@ if __name__ == "__main__":
     print(poem_embeddings.shape)
     poems = [p['beyt'] for p in pe]
 
-    def gradio_make_predictions(text):
-        beyts = predict_poems_from_text(model, poem_embeddings, text, poems, n=10)
+    def gradio_make_predictions(image):
+        beyts = predict_poems_from_image(model, poem_embeddings, image, poems, n=10)
         return "\n".join(beyts)
 
     CFG.batch_size = 512
-    # print(poem_embeddings[0])
-    # with open('poem_embeddings.json'.format(CFG.poem_encoder_model, CFG.text_encoder_model),'w', encoding="utf-8") as f:
-    #     f.write(json.dumps(poem_embeddings, indent= 4))
 
-    text_input = gr.Textbox(label = "Enter the text to find poem beyts for")
+    image_input = gr.Image(type="filepath")
     output = gr.Textbox()
 
-    app = gr.Interface(fn = gradio_make_predictions, inputs=text_input, outputs=output)
+    app = gr.Interface(fn = gradio_make_predictions, inputs=image_input, outputs=output)
     app.launch()

config.py

@@ -103,7 +103,7 @@ image_encoder_weights_load_path = ""
 image_encoder_weights_save_path = "{}_best_image_encoder.pt".format(image_encoder_model)
 image_embedding = 2048   # embedding dim of image encoder's output (for one token)
 # keep this an empty string if you want to use a freshly initialized projection module. else give the path to projection model
-image_projection_load_path = ""
+image_projection_load_path = os.path.join(file_dirname, "projections/{}_best_image_projection.pt".format(image_encoder_model))
 # path to save projection to
 image_projection_save_path = "{}_best_image_projection.pt".format(image_encoder_model)
 image_encoder_trainable = False  # if set to false, this encoder's frozen and its weights won't be saved at all.

projections/resnet50_best_image_projection.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:145f6d7ab06dac0f59906a6c62d19cdfaa5e09e8b0c3a5f2e1a3c975f31ca184
+size 12601871