Ritesh-hf commited on
Commit
a68743b
·
0 Parent(s):

initial commit

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .DS_Store +0 -0
  2. .gitattributes +40 -0
  3. .gitignore +14 -0
  4. Dockerfile +31 -0
  5. README.md +10 -0
  6. app.py +628 -0
  7. configs/data/cirr.yaml +22 -0
  8. configs/data/fashioniq-base.yaml +28 -0
  9. configs/data/fashioniq-dress.yaml +4 -0
  10. configs/data/fashioniq-shirt.yaml +4 -0
  11. configs/data/fashioniq-toptee.yaml +4 -0
  12. configs/data/webvid-covr.yaml +26 -0
  13. configs/data/webvid-covr_rule-based.yaml +26 -0
  14. configs/experiment/cirr.yaml +13 -0
  15. configs/experiment/covr_hard-negatives.yaml +6 -0
  16. configs/experiment/covr_iterate-triplets.yaml +14 -0
  17. configs/experiment/covr_late-fusion.yaml +12 -0
  18. configs/experiment/covr_middle-emb.yaml +14 -0
  19. configs/experiment/covr_only-text.yaml +8 -0
  20. configs/experiment/covr_only-visual.yaml +20 -0
  21. configs/experiment/covr_random-frame.yaml +10 -0
  22. configs/experiment/covr_rule-based.yaml +8 -0
  23. configs/experiment/fiq-dress.yaml +17 -0
  24. configs/experiment/fiq-shirt.yaml +17 -0
  25. configs/experiment/fiq-toptee.yaml +17 -0
  26. configs/machine/default.yaml +16 -0
  27. configs/machine/server.yaml +8 -0
  28. configs/med_config.json +21 -0
  29. configs/model/blip-large.yaml +15 -0
  30. configs/model/blip-large_text.yaml +15 -0
  31. configs/model/blip-large_visual.yaml +15 -0
  32. configs/model/ckpt/blip-l-coco.yaml +3 -0
  33. configs/model/ckpt/cirr-gt.yaml +3 -0
  34. configs/model/ckpt/cirr_ft-covr+gt.yaml +3 -0
  35. configs/model/ckpt/webvid-covr.yaml +3 -0
  36. configs/model/loss/cross_entropy.yaml +2 -0
  37. configs/model/loss/hn_nce.yaml +5 -0
  38. configs/model/optimizer/adamw.yaml +5 -0
  39. configs/model/scheduler/cosine.yaml +6 -0
  40. configs/model/scheduler/step.yaml +5 -0
  41. configs/test.yaml +27 -0
  42. configs/test/all.yaml +6 -0
  43. configs/test/cirr.yaml +15 -0
  44. configs/test/fashioniq-dress.yaml +18 -0
  45. configs/test/fashioniq-shirt.yaml +18 -0
  46. configs/test/fashioniq-toptee.yaml +18 -0
  47. configs/test/fashioniq.yaml +4 -0
  48. configs/test/main.yaml +3 -0
  49. configs/test/webvid-covr.yaml +20 -0
  50. configs/test/webvid-covr_text.yaml +20 -0
.DS_Store ADDED
Binary file (6.15 kB). View file
 
.gitattributes ADDED
@@ -0,0 +1,40 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ *.7z filter=lfs diff=lfs merge=lfs -text
2
+ *.arrow filter=lfs diff=lfs merge=lfs -text
3
+ *.bin filter=lfs diff=lfs merge=lfs -text
4
+ *.bz2 filter=lfs diff=lfs merge=lfs -text
5
+ *.ckpt filter=lfs diff=lfs merge=lfs -text
6
+ *.ftz filter=lfs diff=lfs merge=lfs -text
7
+ *.gz filter=lfs diff=lfs merge=lfs -text
8
+ *.h5 filter=lfs diff=lfs merge=lfs -text
9
+ *.joblib filter=lfs diff=lfs merge=lfs -text
10
+ *.lfs.* filter=lfs diff=lfs merge=lfs -text
11
+ *.mlmodel filter=lfs diff=lfs merge=lfs -text
12
+ *.model filter=lfs diff=lfs merge=lfs -text
13
+ *.msgpack filter=lfs diff=lfs merge=lfs -text
14
+ *.npy filter=lfs diff=lfs merge=lfs -text
15
+ *.npz filter=lfs diff=lfs merge=lfs -text
16
+ *.onnx filter=lfs diff=lfs merge=lfs -text
17
+ *.ot filter=lfs diff=lfs merge=lfs -text
18
+ *.parquet filter=lfs diff=lfs merge=lfs -text
19
+ *.pb filter=lfs diff=lfs merge=lfs -text
20
+ *.pickle filter=lfs diff=lfs merge=lfs -text
21
+ *.pkl filter=lfs diff=lfs merge=lfs -text
22
+ *.pt filter=lfs diff=lfs merge=lfs -text
23
+ *.pth filter=lfs diff=lfs merge=lfs -text
24
+ *.rar filter=lfs diff=lfs merge=lfs -text
25
+ *.safetensors filter=lfs diff=lfs merge=lfs -text
26
+ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
27
+ *.tar.* filter=lfs diff=lfs merge=lfs -text
28
+ *.tar filter=lfs diff=lfs merge=lfs -text
29
+ *.tflite filter=lfs diff=lfs merge=lfs -text
30
+ *.tgz filter=lfs diff=lfs merge=lfs -text
31
+ *.wasm filter=lfs diff=lfs merge=lfs -text
32
+ *.xz filter=lfs diff=lfs merge=lfs -text
33
+ *.zip filter=lfs diff=lfs merge=lfs -text
34
+ *.zst filter=lfs diff=lfs merge=lfs -text
35
+ *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ datasets/sidechef/recipes.json filter=lfs diff=lfs merge=lfs -text
37
+ datasets/sidechef/my_recipes.json filter=lfs diff=lfs merge=lfs -text
38
+ datasets/ filter=lfs diff=lfs merge=lfs -text
39
+ recipes.json filter=lfs diff=lfs merge=lfs -text
40
+ recipe_descriptions.json filter=lfs diff=lfs merge=lfs -text
.gitignore ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ outputs/
2
+ datasets/sidechef/images
3
+ datasets/sidechef/sample_images
4
+ datasets/sidechef/my_tags.json
5
+ datasets/sidechef/tag_categories.json
6
+ datasets/sidechef/tags.json
7
+ launching
8
+ annotation/
9
+ .vscode/
10
+ bert-base-uncased/
11
+ delete*
12
+ __pycache__/
13
+ env/
14
+ .env
Dockerfile ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Read the doc: https://huggingface.co/docs/hub/spaces-sdks-docker
2
+ # you will also find guides on how best to write your Dockerfile
3
+
4
+ FROM python:3.9
5
+
6
+ # Install required system dependencies for OpenCV
7
+ RUN apt-get update && apt-get install -y \
8
+ libgl1-mesa-glx \
9
+ libglib2.0-0 \
10
+ libsm6 \
11
+ libxext6 \
12
+ libxrender-dev \
13
+ && rm -rf /var/lib/apt/lists/*
14
+
15
+ # Create a non-root user to run the app
16
+ RUN useradd -m -u 1000 user
17
+ USER user
18
+ ENV PATH="/home/user/.local/bin:$PATH"
19
+
20
+ # Set the working directory
21
+ WORKDIR /app
22
+
23
+ # Copy the requirements file and install dependencies
24
+ COPY --chown=user ./requirements.txt requirements.txt
25
+ RUN pip install --no-cache-dir --upgrade -r requirements.txt
26
+
27
+ # Copy the rest of the application code
28
+ COPY --chown=user . /app
29
+
30
+ # Set the command to start the application
31
+ CMD ["gunicorn", "-b", "0.0.0.0:7860", "-k", "eventlet", "app:app"]
README.md ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ title: Nutrigenics Flask Chatbot
3
+ emoji: 📉
4
+ colorFrom: blue
5
+ colorTo: gray
6
+ sdk: docker
7
+ pinned: false
8
+ ---
9
+
10
+ Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
app.py ADDED
@@ -0,0 +1,628 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import pandas as pd
2
+ import json
3
+ from PIL import Image
4
+ import numpy as np
5
+
6
+ import os
7
+ from pathlib import Path
8
+
9
+ import torch
10
+ import torch.nn.functional as F
11
+
12
+ # from src.data.embs import ImageDataset
13
+ from src.model.blip_embs import blip_embs
14
+ from src.data.transforms import transform_test
15
+
16
+ from transformers import StoppingCriteria, StoppingCriteriaList, TextIteratorStreamer
17
+ import gradio as gr
18
+ # import spaces
19
+
20
+ from langchain.chains import ConversationChain
21
+ from langchain_community.chat_message_histories import ChatMessageHistory
22
+ from langchain_core.runnables import RunnableWithMessageHistory
23
+ from langchain_core.output_parsers import StrOutputParser
24
+ from langchain_core.prompts import ChatPromptTemplate
25
+ from langchain_groq import ChatGroq
26
+
27
+ from dotenv import load_dotenv
28
+ from flask import Flask, request, render_template
29
+ from flask_cors import CORS
30
+ from flask_socketio import SocketIO, emit
31
+
32
+
33
+ # GROQ_API_KEY = os.getenv("GROQ_API_KEY")
34
+ GROQ_API_KEY = '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'
35
+ load_dotenv(".env")
36
+ USER_AGENT = os.getenv("USER_AGENT")
37
+ GROQ_API_KEY = os.getenv("GROQ_API_KEY")
38
+ SECRET_KEY = os.getenv("SECRET_KEY")
39
+
40
+
41
+ # Set environment variables
42
+ os.environ['USER_AGENT'] = USER_AGENT
43
+ os.environ["GROQ_API_KEY"] = GROQ_API_KEY
44
+ os.environ["TOKENIZERS_PARALLELISM"] = 'true'
45
+
46
+ # Initialize Flask app and SocketIO with CORS
47
+ app = Flask(__name__)
48
+ CORS(app)
49
+ app.config['MAX_CONTENT_LENGTH'] = 1e7
50
+ app.config['SESSION_COOKIE_SECURE'] = True # Use HTTPS
51
+ app.config['SESSION_COOKIE_HTTPONLY'] = True
52
+ app.config['SESSION_COOKIE_SAMESITE'] = 'Lax'
53
+ socketio = SocketIO(app, cors_allowed_origins="*", logger=True, max_http_buffer_size=1e7)
54
+ app.config['SECRET_KEY'] = SECRET_KEY
55
+
56
+ import pandas as pd
57
+ from PIL import Image
58
+ import numpy as np
59
+ import os
60
+
61
+ import torch
62
+ import torch.nn.functional as F
63
+
64
+ # from src.data.embs import ImageDataset
65
+ from src.model.blip_embs import blip_embs
66
+ from src.data.transforms import transform_test
67
+
68
+ from transformers import StoppingCriteria, StoppingCriteriaList, TextIteratorStreamer
69
+ import gradio as gr
70
+ # import spaces
71
+
72
+ from langchain.chains import ConversationChain
73
+ from langchain_community.chat_message_histories import ChatMessageHistory
74
+ from langchain_core.runnables import RunnableWithMessageHistory
75
+ from langchain_core.output_parsers import StrOutputParser
76
+ from langchain_core.prompts import ChatPromptTemplate
77
+ from langchain_groq import ChatGroq
78
+
79
+ from dotenv import load_dotenv
80
+ from flask import Flask, request, render_template
81
+ from flask_cors import CORS
82
+ from flask_socketio import SocketIO, emit
83
+
84
+ import json
85
+ from openai import OpenAI
86
+
87
+ # GROQ_API_KEY = os.getenv("GROQ_API_KEY")
88
+ load_dotenv(".env")
89
+ USER_AGENT = os.getenv("USER_AGENT")
90
+ GROQ_API_KEY = os.getenv("GROQ_API_KEY")
91
+ OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
92
+ SECRET_KEY = os.getenv("SECRET_KEY")
93
+
94
+ # Set environment variables
95
+ os.environ['USER_AGENT'] = USER_AGENT
96
+ os.environ["GROQ_API_KEY"] = GROQ_API_KEY
97
+ os.environ['OPENAI_API_KEY'] = OPENAI_API_KEY
98
+ os.environ["TOKENIZERS_PARALLELISM"] = 'true'
99
+
100
+ # Initialize Flask app and SocketIO with CORS
101
+ app = Flask(__name__)
102
+ CORS(app)
103
+ socketio = SocketIO(app, cors_allowed_origins="*", logger=True)
104
+ app.config['SESSION_COOKIE_SECURE'] = True # Use HTTPS
105
+ app.config['SESSION_COOKIE_HTTPONLY'] = True
106
+ app.config['SESSION_COOKIE_SAMESITE'] = 'Lax'
107
+ app.config['SECRET_KEY'] = SECRET_KEY
108
+
109
+ # Initialize LLM
110
+ llm = ChatGroq(model="llama-3.1-8b-instant", temperature=0, max_tokens=1024, max_retries=2)
111
+
112
+ # Initialize Router
113
+ router = ChatGroq(model="llama-3.2-3b-preview", temperature=0, max_tokens=1024, max_retries=2, model_kwargs={"response_format": {"type": "json_object"}})
114
+
115
+ # Initialize Router
116
+ answer_formatter = ChatGroq(model="llama-3.1-8b-instant", temperature=0, max_tokens=1024, max_retries=2)
117
+
118
+ # Initialized recommendation LLM
119
+ client = OpenAI()
120
+
121
+ class StoppingCriteriaSub(StoppingCriteria):
122
+
123
+ def __init__(self, stops=[], encounters=1):
124
+ super().__init__()
125
+ self.stops = stops
126
+
127
+ def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor):
128
+ for stop in self.stops:
129
+ if torch.all(input_ids[:, -len(stop):] == stop).item():
130
+ return True
131
+
132
+ return False
133
+
134
+ device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
135
+
136
+ def get_blip_config(model="base"):
137
+ config = dict()
138
+ if model == "base":
139
+ config[
140
+ "pretrained"
141
+ ] = "https://storage.googleapis.com/sfr-vision-language-research/BLIP/models/model_base_capfilt_large.pth "
142
+ config["vit"] = "base"
143
+ config["batch_size_train"] = 32
144
+ config["batch_size_test"] = 16
145
+ config["vit_grad_ckpt"] = True
146
+ config["vit_ckpt_layer"] = 4
147
+ config["init_lr"] = 1e-5
148
+ elif model == "large":
149
+ config[
150
+ "pretrained"
151
+ ] = "https://storage.googleapis.com/sfr-vision-language-research/BLIP/models/model_large_retrieval_coco.pth"
152
+ config["vit"] = "large"
153
+ config["batch_size_train"] = 16
154
+ config["batch_size_test"] = 32
155
+ config["vit_grad_ckpt"] = True
156
+ config["vit_ckpt_layer"] = 12
157
+ config["init_lr"] = 5e-6
158
+
159
+ config["image_size"] = 384
160
+ config["queue_size"] = 57600
161
+ config["alpha"] = 0.4
162
+ config["k_test"] = 256
163
+ config["negative_all_rank"] = True
164
+
165
+ return config
166
+
167
+ print("Creating model")
168
+ config = get_blip_config("large")
169
+
170
+ model = blip_embs(
171
+ pretrained=config["pretrained"],
172
+ image_size=config["image_size"],
173
+ vit=config["vit"],
174
+ vit_grad_ckpt=config["vit_grad_ckpt"],
175
+ vit_ckpt_layer=config["vit_ckpt_layer"],
176
+ queue_size=config["queue_size"],
177
+ negative_all_rank=config["negative_all_rank"],
178
+ )
179
+
180
+ model = model.to(device)
181
+ model.eval()
182
+ print("Model Loaded !")
183
+ print("="*50)
184
+
185
+ transform = transform_test(384)
186
+
187
+ print("Loading Data")
188
+ df = pd.read_json("datasets/sidechef/my_recipes.json")
189
+
190
+ print("Loading Target Embedding")
191
+ tar_img_feats = []
192
+ for _id in df["id_"].tolist():
193
+ tar_img_feats.append(torch.load("datasets/sidechef/blip-embs-large/{:07d}.pth".format(_id)).unsqueeze(0))
194
+
195
+ tar_img_feats = torch.cat(tar_img_feats, dim=0)
196
+
197
+ class Chat:
198
+
199
+ def __init__(self, model, transform, dataframe, tar_img_feats, device='cuda:0', stopping_criteria=None):
200
+ self.device = device
201
+ self.model = model
202
+ self.transform = transform
203
+ self.df = dataframe
204
+ self.tar_img_feats = tar_img_feats
205
+ self.img_feats = None
206
+ self.target_recipe = None
207
+ self.messages = []
208
+
209
+ if stopping_criteria is not None:
210
+ self.stopping_criteria = stopping_criteria
211
+ else:
212
+ stop_words_ids = [torch.tensor([2]).to(self.device)]
213
+ self.stopping_criteria = StoppingCriteriaList([StoppingCriteriaSub(stops=stop_words_ids)])
214
+
215
+ def encode_image(self, image_path):
216
+ img = Image.fromarray(image_path).convert("RGB")
217
+ img = self.transform(img).unsqueeze(0)
218
+ img = img.to(self.device)
219
+ img_embs = model.visual_encoder(img)
220
+ img_feats = F.normalize(model.vision_proj(img_embs[:, 0, :]), dim=-1).cpu()
221
+
222
+ self.img_feats = img_feats
223
+
224
+ self.get_target(self.img_feats, self.tar_img_feats)
225
+
226
+ def get_target(self, img_feats, tar_img_feats) :
227
+ score = (img_feats @ tar_img_feats.t()).squeeze(0).cpu().detach().numpy()
228
+ index = np.argsort(score)[::-1][0]
229
+ self.target_recipe = df.iloc[index]
230
+
231
+ def ask(self):
232
+ return json.dumps(self.target_recipe.to_json())
233
+
234
+
235
+
236
+ chat = Chat(model,transform,df,tar_img_feats, device)
237
+ print("Chat Initialized !")
238
+
239
+
240
+ def answer_generator(formated_input, session_id):
241
+ # QA system prompt and chain
242
+ qa_system_prompt = """
243
+ You are an AI assistant developed by Nutrigenics AI, specializing in intelligent recipe information retrieval and recipe suggestions. Your purpose is to help users by recommending recipes, providing detailed nutritional values, listing ingredients, offering step-by-step cooking instructions, and filtering recipes based on provide context ans user query.
244
+ Operational Guidelines:
245
+ 1. Input Structure:
246
+ - Context: You may receive contextual information related to recipes, such as specific data sets, user preferences, dietary restrictions, or previously selected dishes.
247
+ - User Query: Users will pose questions or requests related to recipes, nutritional information, ingredient substitutions, cooking instructions, and more.
248
+ 2. Response Strategy:
249
+ - Utilize Provided Context: If the context contains relevant information that addresses the user's query, base your response on this provided data to ensure accuracy and relevance.
250
+ - Respond to User Query Directly: If the context does not contain the necessary information to answer the user's query, kindly state that you do not have require information.
251
+ Core Functionalities:
252
+ - Nutritional Information: Accurately provide nutritional values for each recipe, including calories, macronutrients (proteins, fats, carbohydrates), and essential vitamins and minerals, using contextual data when available.
253
+ - Ingredient Details: List all ingredients required for recipes, including substitute options for dietary restrictions or ingredient availability, utilizing context when relevant.
254
+ - Step-by-Step Cooking Instructions: Deliver clear, easy-to-follow instructions for preparing and cooking meals, informed by any provided contextual data.
255
+ - Recipe Recommendations: Suggest dishes based on user preferences, dietary restrictions, available ingredients, and contextual data if provided.
256
+ Additional Instructions:
257
+ - Precision and Personalization: Always aim to provide precise, personalized, and relevant information to users based on both the provided context and their specific queries.
258
+ - Clarity and Coherence: Ensure that all responses are clear, well-structured, and easy to understand, facilitating a seamless user experience.
259
+ - Substitute Suggestions: When suggesting ingredient substitutes, consider user preferences and dietary restrictions outlined in the context or user query.
260
+ - Dynamic Adaptation: Adapt your responses dynamically based on whether the context is relevant to the user's current request, ensuring optimal use of available information.
261
+ Don't mention about context in the response, format the answer in a natural and friendly way.
262
+ Context:
263
+ {context}
264
+ """
265
+ qa_prompt = ChatPromptTemplate.from_messages(
266
+ [
267
+ ("system", qa_system_prompt),
268
+ ("human", "{input}")
269
+ ]
270
+ )
271
+
272
+ # Create the base chain
273
+ base_chain = qa_prompt | llm | StrOutputParser()
274
+
275
+ # Wrap the chain with message history
276
+ question_answer_chain = RunnableWithMessageHistory(
277
+ base_chain,
278
+ lambda session_id: ChatMessageHistory(), # This creates a new history for each session
279
+ input_messages_key="input",
280
+ history_messages_key="chat_history"
281
+ )
282
+
283
+ response = question_answer_chain.invoke(formated_input, config={"configurable": {"session_id": session_id}})
284
+
285
+ return response
286
+
287
+
288
+
289
+ ### Router
290
+ import json
291
+ from langchain_core.messages import HumanMessage, SystemMessage
292
+
293
+ def router_node(query):
294
+ # Prompt
295
+ router_instructions = """You are an expert at determining the appropriate task for a user’s question based on chat history and the current query context. You have two available tasks:
296
+
297
+ 1. Retrieval: Fetch information based on user's chat history and current query.
298
+ 2. Recommendation/Suggestion: Recommend recipes to users based on the query.
299
+
300
+ Return a JSON response with a single key named “task” indicating either “retrieval” or “recommendation” based on your decision.
301
+ """
302
+ response = router.invoke(
303
+ [SystemMessage(content=router_instructions)]
304
+ + [
305
+ HumanMessage(
306
+ content=query
307
+ )
308
+ ]
309
+ )
310
+ res = json.loads(response.content)
311
+ return res['task']
312
+
313
+ def recommendation_node(query):
314
+ prompt = """
315
+ You are a helpful assistant that writes Python code to filter recipes from a JSON filr based o the user query. \n
316
+ JSON file path = 'recipes.json' \n
317
+ The JSON file is a list of recipes with the following structure: \n
318
+ {
319
+ "recipe_name": string,
320
+ "recipe_time": integer,
321
+ "recipe_yields": string,
322
+ "recipe_ingredients": list of ingredients,
323
+ "recipe_instructions": list of instruections,
324
+ "recipe_image": string,
325
+ "blogger": string,
326
+ "recipe_nutrients": JSON object with key value pairs such as "protein: 10g",
327
+ "tags": list of tags related to recipe
328
+ } \n
329
+
330
+ Here is the example of an recipe json object from the JSON data: \n
331
+ {
332
+ "recipe_name": "Asian Potato Salad with Seven Minute Egg",
333
+ "recipe_time": 0,
334
+ "recipe_yields": "4 servings",
335
+ "recipe_ingredients": [
336
+ "2 1/2 cup Multi-Colored Fingerling Potato",
337
+ "3/4 cup Celery",
338
+ "1/4 cup Red Onion",
339
+ "2 tablespoon Fresh Parsley",
340
+ "1/3 cup Mayonnaise",
341
+ "1 tablespoon Chili Garlic Sauce",
342
+ "1 teaspoon Hoisin Sauce",
343
+ "1 splash Soy Sauce",
344
+ "to taste Salt",
345
+ "to taste Ground Black Pepper",
346
+ "4 Egg"
347
+ ],
348
+ "recipe_instructions": "Fill a large stock pot with water.\nAdd the Multi-Colored Fingerling Potato (2 1/2 cup) and bring water to a boil. Boil the potatoes for 20 minutes or until fork tender.\nDrain the potatoes and let them cool completely.\nMeanwhile, mix together in a small bowl Mayonnaise (1/3 cup), Chili Garlic Sauce (1 tablespoon), Hoisin Sauce (1 teaspoon), and Soy Sauce (1 splash).\nTo make the Egg (4), fill a stock pot with water and bring to a boil Gently add the eggs to the water and set a timer for seven minutes.\nThen move the eggs to an ice bath to cool completely. Once cooled, crack the egg slightly and remove the shell. Slice in half when ready to serve.\nNext, halve the cooled potatoes and place into a large serving bowl. Add the Ground Black Pepper (to taste), Celery (3/4 cup), Red Onion (1/4 cup), and mayo mixture. Toss to combine adding Salt (to taste) and Fresh Parsley (2 tablespoon).\nTop with seven minute eggs and serve. Enjoy!",
349
+ "recipe_image": "https://www.sidechef.com/recipe/eeeeeceb-493e-493d-8273-66c800821b13.jpg?d=1408x1120",
350
+ "blogger": "sidechef.com",
351
+ "recipe_nutrients": {
352
+ "calories": "80 calories",
353
+ "proteinContent": "2.1 g",
354
+ "fatContent": "6.2 g",
355
+ "carbohydrateContent": "3.9 g",
356
+ "fiberContent": "0.5 g",
357
+ "sugarContent": "0.4 g",
358
+ "sodiumContent": "108.0 mg",
359
+ "saturatedFatContent": "1.2 g",
360
+ "transFatContent": "0.0 g",
361
+ "cholesterolContent": "47.4 mg",
362
+ "unsaturatedFatContent": "3.8 g"
363
+ },
364
+ "tags": [
365
+ "Salad",
366
+ "Lunch",
367
+ "Brunch",
368
+ "Appetizers",
369
+ "Side Dish",
370
+ "Budget-Friendly",
371
+ "Vegetarian",
372
+ "Pescatarian",
373
+ "Eggs",
374
+ "Potatoes",
375
+ "Dairy-Free",
376
+ "Shellfish-Free"
377
+ ]
378
+ } \n
379
+
380
+ Based on the user query, provide a Python function to filter the JSON data. The output of the function should be a list of json objects. \n
381
+
382
+ Recipe filtering instructions:
383
+ - If a user asked for the highest nutrient recipe such as "high protein or high calories" then filtered recipes should be the top highest recipes from all the recipes with high nutrient.
384
+ - sort or rearrange recipes based which recipes are more appropriate for the user.
385
+
386
+ Your output instructions:
387
+ - The function name should be filter_recipes. The input to the function should be file name.
388
+ - The length of output recipes should not be more than 6.
389
+ - Only give me output function. Do not call the function.
390
+ - Give the python function as a key named "code" in a json format.
391
+ - Do not include any other text with the output, only give python code.
392
+ - If you do not follow the above given instructions, the chat may be terminated.
393
+ """
394
+ max_tries = 3
395
+ while True:
396
+ try:
397
+ # llm = ChatGroq(model="llama-3.1-8b-instant", temperature=0, max_tokens=1024, max_retries=2, model_kwargs={"response_format": {"type": "json_object"}})
398
+ response = client.chat.completions.create(
399
+ model="gpt-4o-mini",
400
+ messages=[
401
+ {"role": "system", "content": prompt},
402
+ {
403
+ "role": "user",
404
+ "content": query
405
+ }
406
+ ]
407
+ )
408
+
409
+ content = response.choices[0].message.content
410
+
411
+ res = json.loads(content)
412
+ script = res['code']
413
+ exec(script, globals())
414
+ filtered_recipes = filter_recipes('recipes.json')
415
+ if len(filtered_recipes) > 0:
416
+ return filtered_recipes
417
+ except Exception as e:
418
+ print(e)
419
+ if max_tries <= 0:
420
+ return []
421
+ else:
422
+ max_tries -= 1
423
+ return filtered_recipes
424
+
425
+
426
+ def answer_formatter_node(question, context):
427
+ prompt = f""" You are an highly clever question-answering assistant trained to provide clear and concise answers based on the user query and provided context.
428
+ Your task is to generated answers for the user query based on the context provided.
429
+ Instructions for your response:
430
+ 1. Directly answer the user query using only the information provided in the context.
431
+ 2. Ensure your response is clear and concise.
432
+ 3. Mention only details related to the recipe, including the recipe name, instructions, nutrients, yield, ingredients, and image.
433
+ 4. Do not include any information that is not related to the recipe context.
434
+
435
+ Please format an answer based on the following user question and context provided:
436
+
437
+ User Question:
438
+ {question}
439
+
440
+ Context:
441
+ {context}
442
+ """
443
+ response = answer_formatter.invoke(
444
+ [SystemMessage(content=prompt)]
445
+ )
446
+ res = response.content
447
+ return res
448
+
449
+ CURR_CONTEXT = ''
450
+
451
+ # @spaces.GPU
452
+ def get_answer(image=[], message='', sessionID='abc123'):
453
+ global CURR_CONTEXT
454
+ if len(image) > 0:
455
+ try:
456
+ # Process the image and message here
457
+ device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
458
+ chat = Chat(model,transform,df,tar_img_feats, device)
459
+ chat.encode_image(image)
460
+ data = chat.ask()
461
+ CURR_CONTEXT = data
462
+ formated_input = {
463
+ 'input': message,
464
+ 'context': data
465
+ }
466
+ response = answer_generator(formated_input, session_id=sessionID)
467
+ except Exception as e:
468
+ print(e)
469
+ response = {'content':"An error occurred while processing your request."}
470
+ elif len(image) == 0 and message is not None:
471
+ print("I am here")
472
+ task = router_node(message)
473
+ if task == 'retrieval':
474
+ recipes = recommendation_node(message)
475
+ if not recipes:
476
+ response = {'content':"An error occurred while processing your request."}
477
+ response = answer_formatter_node(message, recipes)
478
+ else:
479
+ formated_input = {
480
+ 'input': message,
481
+ 'context': CURR_CONTEXT
482
+ }
483
+ response = answer_generator(formated_input, session_id=sessionID)
484
+
485
+ return response
486
+
487
+ # Function to handle WebSocket connection
488
+ @socketio.on('ping')
489
+ def handle_connect():
490
+ emit('Ping-return', {'message': 'Connected'}, room=request.sid)
491
+
492
+
493
+ # Function to handle WebSocket connection
494
+ @socketio.on('connect')
495
+ def handle_connect():
496
+ print(f"Client connected: {request.sid}")
497
+ emit('connection_response', {'message': 'Connected successfully.'})
498
+
499
+ # Function to handle WebSocket disconnection
500
+ @socketio.on('disconnect')
501
+ def handle_disconnect():
502
+ print(f"Client disconnected: {request.sid}")
503
+
504
+ import json
505
+ import base64
506
+ from PIL import Image
507
+ from io import BytesIO
508
+ import torchvision.transforms as transforms
509
+
510
+ # Dictionary to store incomplete image data by session
511
+ session_store = {}
512
+
513
+ @socketio.on('message')
514
+ def handle_message(data):
515
+ global session_store
516
+ global CURR_CONTEXT
517
+ context = "No data available"
518
+ session_id = request.sid
519
+ if session_id not in session_store:
520
+ session_store[session_id] = {'image_data': b"", 'message': None, 'image_received': False}
521
+
522
+ if 'message' in data:
523
+ session_store[session_id]['message'] = data['message']
524
+
525
+ # Handle image chunk data
526
+ if 'image' in data:
527
+ try:
528
+ # Append the incoming image chunk
529
+ session_store[session_id]['image_data'] += data['image']
530
+
531
+ except Exception as e:
532
+ print(f"Error processing image chunk: {str(e)}")
533
+ emit('response', "An error occurred while receiving the image chunk.", room=session_id)
534
+ return
535
+
536
+ if session_store[session_id]['image_data'] or session_store[session_id]['message']:
537
+ try:
538
+ image_bytes = session_store[session_id]['image_data']
539
+ # print("checkpoint 2")
540
+ if isinstance(image_bytes, str):
541
+ image_bytes = base64.b64decode(image_bytes)
542
+ image = Image.open(BytesIO(image_bytes))
543
+ image_array = np.array(image)
544
+ device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
545
+ chat = Chat(model, transform, df, tar_img_feats, device)
546
+ chat.encode_image(image_array)
547
+ context = chat.ask()
548
+ CURR_CONTEXT = context
549
+ message = data['message']
550
+ formated_input = {
551
+ 'input': message,
552
+ 'context': json.dumps(context)
553
+ }
554
+ # Invoke question_answer_chain and stream the response
555
+ response = answer_generator(formated_input, session_id=session_id)
556
+ emit('response', response, room=session_id)
557
+
558
+ except Exception as e:
559
+ print(f"Error processing image or message: {str(e)}")
560
+ emit('response', "An error occurred while processing your request.", room=session_id)
561
+ return
562
+ finally:
563
+ # Clear session data after processing
564
+ session_store.pop(session_id, None)
565
+ else:
566
+ message = data['message']
567
+ task = router_node(message)
568
+ print(task)
569
+ if task == 'retrieval':
570
+ formated_input = {
571
+ 'input': message,
572
+ 'context': json.dumps(CURR_CONTEXT)
573
+ }
574
+ response = answer_generator(formated_input, session_id=session_id)
575
+ emit('response', response, room=session_id)
576
+ else:
577
+ response = recommendation_node(message)
578
+ # response = answer_formatter_node(message, recipes)
579
+ if response is None:
580
+ response = {'content':"An error occurred while processing your request."}
581
+
582
+ emit('json_response', response, room=session_id)
583
+ session_store.pop(session_id, None)
584
+
585
+
586
+
587
+ import requests
588
+ from PIL import Image
589
+ import numpy as np
590
+ from io import BytesIO
591
+
592
+ def download_image_to_numpy(url):
593
+ # Send a GET request to the URL to download the image
594
+ response = requests.get(url)
595
+
596
+ # Check if the request was successful
597
+ if response.status_code == 200:
598
+ # Open the image using PIL and convert it to RGB format
599
+ image = Image.open(BytesIO(response.content)).convert('RGB')
600
+
601
+ # Convert the image to a NumPy array
602
+ image_array = np.array(image)
603
+
604
+ return image_array
605
+ else:
606
+ raise Exception(f"Failed to download image. Status code: {response.status_code}")
607
+
608
+ @socketio.on('example')
609
+ def handle_message(data):
610
+ img_url = data['img_url']
611
+ message = data['message']
612
+ session_id = request.sid
613
+ image_array = download_image_to_numpy(img_url)
614
+ response = get_answer(image=image_array, message=message, sessionID=request.sid)
615
+ emit('response', response, room=session_id)
616
+ return response
617
+
618
+
619
+
620
+
621
+ # Home route
622
+ @app.route("/")
623
+ def index_view():
624
+ return render_template('chat.html')
625
+
626
+ # Main function to run the app
627
+ if __name__ == '__main__':
628
+ socketio.run(app, debug=True)
configs/data/cirr.yaml ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ dataname: cirr
2
+ _target_: src.data.cirr.CIRRDataModule
3
+
4
+ # Paths
5
+ dataset_dir: ${paths.datasets_dir}/CIRR
6
+
7
+ batch_size: ${machine.batch_size}
8
+ num_workers: ${machine.num_workers}
9
+
10
+ annotation:
11
+ train: ${paths.work_dir}/annotation/cirr/cap.rc2.train.json
12
+ val: ${paths.work_dir}/annotation/cirr/cap.rc2.val.json
13
+
14
+ img_dirs:
15
+ train: ${data.dataset_dir}/images/train
16
+ val: ${data.dataset_dir}/images/dev
17
+
18
+ emb_dirs:
19
+ train: ${data.dataset_dir}/blip-embs-large/train
20
+ val: ${data.dataset_dir}/blip-embs-large/dev
21
+
22
+ image_size: 384
configs/data/fashioniq-base.yaml ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ dataname: fashioniq-${data.category}
2
+ _target_: src.data.fashioniq.FashionIQDataModule
3
+
4
+ # Paths
5
+ dataset_dir: ${paths.datasets_dir}/fashion-iq
6
+
7
+ batch_size: ${machine.batch_size}
8
+ num_workers: ${machine.num_workers}
9
+
10
+ annotation:
11
+ train: ${paths.work_dir}/annotation/fashion-iq/cap.${data.category}.train.json
12
+ val: ${paths.work_dir}/annotation/fashion-iq/cap.${data.category}.val.json
13
+
14
+ targets:
15
+ train: ${paths.work_dir}/annotation/fashion-iq/split.${data.category}.train.json
16
+ val: ${paths.work_dir}/annotation/fashion-iq/split.${data.category}.val.json
17
+
18
+ img_dirs:
19
+ train: ${data.dataset_dir}/images/
20
+ val: ${data.dataset_dir}/images/
21
+
22
+ emb_dirs:
23
+ train: ${data.dataset_dir}/blip-embs-large/
24
+ val: ${data.dataset_dir}/blip-embs-large/
25
+
26
+ image_size: 384
27
+
28
+ category: ???
configs/data/fashioniq-dress.yaml ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ defaults:
2
+ - fashioniq-base.yaml
3
+
4
+ category: dress
configs/data/fashioniq-shirt.yaml ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ defaults:
2
+ - fashioniq-base.yaml
3
+
4
+ category: shirt
configs/data/fashioniq-toptee.yaml ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ defaults:
2
+ - fashioniq-base.yaml
3
+
4
+ category: toptee
configs/data/webvid-covr.yaml ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ dataname: webvid-covr
2
+ _target_: src.data.webvid_covr.WebVidCoVRDataModule
3
+
4
+ image_size: 384
5
+ iterate: "pth2"
6
+ vid_query_method: middle
7
+ vid_frames: 1
8
+ emb_pool: query
9
+
10
+ # Paths
11
+ dataset_dir: ${paths.datasets_dir}/WebVid
12
+
13
+ batch_size: ${machine.batch_size}
14
+ num_workers: ${machine.num_workers}
15
+
16
+ annotation:
17
+ train: ${paths.work_dir}/annotation/webvid-covr/webvid2m-covr_train.csv
18
+ val: ${paths.work_dir}/annotation/webvid-covr/webvid8m-covr_val.csv
19
+
20
+ vid_dirs:
21
+ train: ${data.dataset_dir}/2M/train
22
+ val: ${data.dataset_dir}/8M/train
23
+
24
+ emb_dirs:
25
+ train: ${data.dataset_dir}/2M/blip-vid-embs-${model.model.vit}-all
26
+ val: ${data.dataset_dir}/8M/blip-vid-embs-${model.model.vit}-all
configs/data/webvid-covr_rule-based.yaml ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ dataname: webvid-covr-rule-based
2
+ _target_: src.data.webvid_covr_rulebased.WebVidCoVRDataModuleRuleBased
3
+
4
+ image_size: 384
5
+ iterate: "pth2"
6
+ vid_query_method: middle
7
+ vid_frames: 1
8
+ emb_pool: query
9
+
10
+ # Paths
11
+ dataset_dir: ${paths.datasets_dir}/WebVid
12
+
13
+ batch_size: ${machine.batch_size}
14
+ num_workers: ${machine.num_workers}
15
+
16
+ annotation:
17
+ train: ${paths.work_dir}/annotation/webvid-covr/webvid2m-covr_train.csv
18
+ val: ${paths.work_dir}/annotation/webvid-covr/webvid8m-covr_val.csv
19
+
20
+ vid_dirs:
21
+ train: ${data.dataset_dir}/2M/train
22
+ val: ${data.dataset_dir}/8M/train
23
+
24
+ emb_dirs:
25
+ train: ${data.dataset_dir}/2M/blip-vid-embs-${model.model.vit}-all
26
+ val: ${data.dataset_dir}/8M/blip-vid-embs-${model.model.vit}-all
configs/experiment/cirr.yaml ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # @package _global_
2
+
3
+ defaults:
4
+ - override /data: cirr.yaml
5
+ - override /test: cirr.yaml
6
+ # - override /model/ckpt: webvid-covr.yaml
7
+
8
+ model:
9
+ optimizer:
10
+ lr: 1e-4
11
+
12
+ trainer:
13
+ max_epochs: 6
configs/experiment/covr_hard-negatives.yaml ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ # @package _global_
2
+
3
+ defaults:
4
+ - override /data: webvid-covr.yaml
5
+ - override /test: main.yaml
6
+ - override /model/loss: cross_entropy
configs/experiment/covr_iterate-triplets.yaml ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # @package _global_
2
+
3
+ defaults:
4
+ - override /data: webvid-covr.yaml
5
+ - override /test: main.yaml
6
+
7
+ run_name: "iterate-triplets"
8
+
9
+ data:
10
+ iterate: "triplets"
11
+
12
+ trainer:
13
+ max_epochs: 2
14
+ print_interval: 1
configs/experiment/covr_late-fusion.yaml ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # @package _global_
2
+
3
+ defaults:
4
+ - override /data: webvid-covr.yaml
5
+ - override /model: blip-large_late-fusion.yaml
6
+ - override /test: webvid-covr_late-fusion.yaml
7
+
8
+ val: False
9
+
10
+ model:
11
+ optimizer:
12
+ lr: 1e-4
configs/experiment/covr_middle-emb.yaml ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # @package _global_
2
+
3
+ run_name: "middle_emb"
4
+
5
+ defaults:
6
+ - override /data: webvid-covr.yaml
7
+ - override /test: webvid-covr.yaml
8
+
9
+ data:
10
+ emb_pool: "middle"
11
+
12
+ test:
13
+ webvid_covr:
14
+ emb_pool: "middle"
configs/experiment/covr_only-text.yaml ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ # @package _global_
2
+
3
+ defaults:
4
+ - override /data: webvid-covr.yaml
5
+ - override /test: webvid-covr_text.yaml
6
+ - override /model: blip-large_text.yaml
7
+
8
+ val: False
configs/experiment/covr_only-visual.yaml ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # @package _global_
2
+
3
+ defaults:
4
+ - override /data: webvid-covr.yaml
5
+ - override /test: webvid-covr_visual.yaml
6
+ - override /model: blip-large_visual.yaml
7
+
8
+ val: False
9
+
10
+ run_name: only-visual
11
+
12
+ machine:
13
+ batch_size: 64 # We have to reduce the learning rate because we are training the ViT
14
+
15
+ model:
16
+ optimizer:
17
+ lr: 0.125e-4 # We have to reduce the learning rate because we are reducing the batch size
18
+
19
+ data:
20
+ emb_pool: mean
configs/experiment/covr_random-frame.yaml ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ # @package _global_
2
+
3
+ run_name: "random-frame"
4
+
5
+ defaults:
6
+ - override /data: webvid-covr.yaml
7
+ - override /test: webvid-covr.yaml
8
+
9
+ data:
10
+ vid_query_method: "random"
configs/experiment/covr_rule-based.yaml ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ # @package _global_
2
+
3
+ defaults:
4
+ - override /data: webvid-covr_rule-based.yaml
5
+ - override /test: main.yaml
6
+
7
+ trainer:
8
+ print_interval: 2
configs/experiment/fiq-dress.yaml ADDED
@@ -0,0 +1,17 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # @package _global_
2
+
3
+ defaults:
4
+ - override /data: fashioniq-dress.yaml
5
+ - override /test: fashioniq-dress.yaml
6
+ - override /model/ckpt: webvid-covr.yaml
7
+
8
+ model:
9
+ optimizer:
10
+ lr: 1e-4
11
+
12
+ machine:
13
+ batch_size: 256
14
+
15
+ trainer:
16
+ max_epochs: 6
17
+ print_interval: 2
configs/experiment/fiq-shirt.yaml ADDED
@@ -0,0 +1,17 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # @package _global_
2
+
3
+ defaults:
4
+ - override /data: fashioniq-shirt.yaml
5
+ - override /test: fashioniq-shirt.yaml
6
+ - override /model/ckpt: webvid-covr.yaml
7
+
8
+ model:
9
+ optimizer:
10
+ lr: 1e-4
11
+
12
+ machine:
13
+ batch_size: 256
14
+
15
+ trainer:
16
+ max_epochs: 6
17
+ print_interval: 2
configs/experiment/fiq-toptee.yaml ADDED
@@ -0,0 +1,17 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # @package _global_
2
+
3
+ defaults:
4
+ - override /data: fashioniq-toptee.yaml
5
+ - override /test: fashioniq-toptee.yaml
6
+ - override /model/ckpt: webvid-covr.yaml
7
+
8
+ model:
9
+ optimizer:
10
+ lr: 1e-4
11
+
12
+ machine:
13
+ batch_size: 256
14
+
15
+ trainer:
16
+ max_epochs: 6
17
+ print_interval: 2
configs/machine/default.yaml ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # path to root directory
2
+ root_dir: "."
3
+
4
+ # path to working directory
5
+ work_dir: ${hydra:runtime.cwd}
6
+
7
+ # path to output directory, created dynamically by hydra
8
+ # path generation pattern is specified in `configs/hydra/default.yaml`
9
+ # use it to store all files generated during the run, like ckpts and metrics
10
+ output_dir: ${hydra:runtime.output_dir}
11
+
12
+ # path to dataset directory
13
+ datasets_dir: ${hydra:runtime.cwd}/datasets/
14
+
15
+ # path to logging directory
16
+ log_dir: ${paths.root_dir}/logs/
configs/machine/server.yaml ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ name: server
2
+
3
+ # specific attributes to this machine
4
+ batch_size: 512
5
+ num_workers: 8
6
+
7
+ defaults:
8
+ - default@paths
configs/med_config.json ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "BertModel"
4
+ ],
5
+ "attention_probs_dropout_prob": 0.1,
6
+ "hidden_act": "gelu",
7
+ "hidden_dropout_prob": 0.1,
8
+ "hidden_size": 768,
9
+ "initializer_range": 0.02,
10
+ "intermediate_size": 3072,
11
+ "layer_norm_eps": 1e-12,
12
+ "max_position_embeddings": 512,
13
+ "model_type": "bert",
14
+ "num_attention_heads": 12,
15
+ "num_hidden_layers": 12,
16
+ "pad_token_id": 0,
17
+ "type_vocab_size": 2,
18
+ "vocab_size": 30524,
19
+ "encoder_width": 768,
20
+ "add_cross_attention": true
21
+ }
configs/model/blip-large.yaml ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ modelname: blip-large
2
+ _target_: src.model.blip_cir.blip_cir
3
+
4
+ ckpt_path: ${model.ckpt.path}
5
+
6
+ model:
7
+ _target_: src.model.blip_cir.BLIPCir
8
+ med_config: ${paths.work_dir}/configs/med_config.json
9
+ image_size: ${data.image_size}
10
+ vit: "large"
11
+ vit_grad_ckpt: True
12
+ vit_ckpt_layer: 12
13
+ embed_dim: 256
14
+ train_vit: False
15
+ loss: ${model.loss}
configs/model/blip-large_text.yaml ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ modelname: blip-large-text
2
+ _target_: src.model.blip_cir_text.blip_cir_text
3
+
4
+ ckpt_path: ${model.ckpt.path}
5
+
6
+ model:
7
+ _target_: src.model.blip_cir_text.BLIPCirTextOnly
8
+ med_config: ${paths.work_dir}/configs/med_config.json
9
+ image_size: ${data.image_size}
10
+ vit: "large"
11
+ vit_grad_ckpt: True
12
+ vit_ckpt_layer: 12
13
+ embed_dim: 256
14
+ train_vit: False
15
+ loss: ${model.loss}
configs/model/blip-large_visual.yaml ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ modelname: blip-large-visual
2
+ _target_: src.model.blip_cir_visual.blip_cir_visual
3
+
4
+ ckpt_path: ${model.ckpt.path}
5
+
6
+ model:
7
+ _target_: src.model.blip_cir_visual.BLIPCirVisualOnly
8
+ med_config: ${paths.work_dir}/configs/med_config.json
9
+ image_size: ${data.image_size}
10
+ vit: "large"
11
+ vit_grad_ckpt: True
12
+ vit_ckpt_layer: 12
13
+ embed_dim: 256
14
+ train_vit: True
15
+ loss: ${model.loss}
configs/model/ckpt/blip-l-coco.yaml ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ name: blip-l-coco
2
+
3
+ path: "https://storage.googleapis.com/sfr-vision-language-research/BLIP/models/model_large_retrieval_coco.pth"
configs/model/ckpt/cirr-gt.yaml ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ name: cirr-gt
2
+
3
+ path: ${paths.work_dir}/outputs/cirr/blip-large/blip-l-coco/tv-False_loss-hnnce_lr-1e-05/base/ckpt_4.ckpt
configs/model/ckpt/cirr_ft-covr+gt.yaml ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ name: cirr_ft-covr+gt
2
+
3
+ path: ${paths.work_dir}/outputs/cirr/blip-large/webvid-covr/tv-False_loss-hnnce_lr-0.0001/base/ckpt_5.ckpt
configs/model/ckpt/webvid-covr.yaml ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ name: webvid-covr
2
+
3
+ path: ${paths.work_dir}/outputs/webvid-covr/blip-large/blip-l-coco/tv-False_loss-hnnce_lr-1e-05/good/ckpt_4.ckpt
configs/model/loss/cross_entropy.yaml ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ _target_: src.model.loss.CrossEntropyLoss
2
+ name: ce
configs/model/loss/hn_nce.yaml ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ _target_: src.model.loss.HardNegativeNCE
2
+ name: hnnce
3
+
4
+ alpha: 1
5
+ beta: 0.5
configs/model/optimizer/adamw.yaml ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ _target_: torch.optim.AdamW
2
+ _partial_: true
3
+
4
+ lr: 1e-05
5
+ weight_decay: 0.05
configs/model/scheduler/cosine.yaml ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ _target_: src.tools.scheduler.CosineSchedule
2
+
3
+ init_lr: ${model.optimizer.lr}
4
+ min_lr: 0
5
+ decay_rate: ${model.optimizer.weight_decay}
6
+ max_epochs: ${trainer.max_epochs}
configs/model/scheduler/step.yaml ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ _target_: src.tools.scheduler.StepSchedule
2
+
3
+ init_lr: ${model.optimizer.lr}
4
+ decay_rate: ${model.optimizer.weight_decay}
5
+ min_lr: 0
configs/test.yaml ADDED
@@ -0,0 +1,27 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ hydra:
2
+ run:
3
+ dir: outputs/test/${model.modelname}/${model.ckpt.name}/${run_name}
4
+ job: # automatically go to the job folder (needed for hydra > 1.2 with new behavior)
5
+ chdir: true
6
+
7
+ # Global configurations shared between different modules
8
+ run_name: base
9
+
10
+ seed: 1234
11
+ logger_level: INFO
12
+
13
+ # Composing nested config with default
14
+ defaults:
15
+ - _self_
16
+ - data: cirr
17
+ - test: all
18
+ - machine: server
19
+ - trainer: gpu
20
+ - model: blip-large
21
+ - model/ckpt: blip-l-coco
22
+ - model/loss: hn_nce
23
+ - trainer/logger: none
24
+
25
+ - experiment: null
26
+
27
+ paths: ${machine.paths}
configs/test/all.yaml ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ defaults:
2
+ - cirr.yaml
3
+ - webvid-covr.yaml
4
+ - fashioniq-dress.yaml
5
+ - fashioniq-shirt.yaml
6
+ - fashioniq-toptee.yaml
configs/test/cirr.yaml ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ cirr:
2
+ dataname: cirr
3
+ _target_: src.data.cirr.CIRRTestDataModule
4
+
5
+ test:
6
+ _target_: src.test.cirr.TestCirr
7
+
8
+ batch_size: ${machine.batch_size}
9
+ num_workers: ${machine.num_workers}
10
+
11
+ annotation: ${paths.work_dir}/annotation/cirr/cap.rc2.test1.json
12
+ img_dirs: ${paths.datasets_dir}/CIRR/images/test1
13
+ emb_dirs: ${paths.datasets_dir}/CIRR/blip-embs-large/test1
14
+
15
+ image_size: 384
configs/test/fashioniq-dress.yaml ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ fashioniq-dress:
2
+ dataname: fashioniq-dress
3
+ _target_: src.data.fashioniq.FashionIQTestDataModule
4
+
5
+ batch_size: ${machine.batch_size}
6
+ num_workers: ${machine.num_workers}
7
+
8
+ annotation: ${paths.work_dir}/annotation/fashion-iq/cap.dress.val.json
9
+ targets: ${paths.work_dir}/annotation/fashion-iq/split.dress.val.json
10
+
11
+ img_dirs: ${paths.datasets_dir}/fashion-iq/images/
12
+ emb_dirs: ${paths.datasets_dir}/fashion-iq/blip-embs-large/
13
+
14
+ image_size: 384
15
+
16
+ test:
17
+ _target_: src.test.fashioniq.TestFashionIQ
18
+ category: dress
configs/test/fashioniq-shirt.yaml ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ fashioniq-shirt:
2
+ dataname: fashioniq-shirt
3
+ _target_: src.data.fashioniq.FashionIQTestDataModule
4
+
5
+ batch_size: ${machine.batch_size}
6
+ num_workers: ${machine.num_workers}
7
+
8
+ annotation: ${paths.work_dir}/annotation/fashion-iq/cap.shirt.val.json
9
+ targets: ${paths.work_dir}/annotation/fashion-iq/split.shirt.val.json
10
+
11
+ img_dirs: ${paths.datasets_dir}/fashion-iq/images/
12
+ emb_dirs: ${paths.datasets_dir}/fashion-iq/blip-embs-large/
13
+
14
+ image_size: 384
15
+
16
+ test:
17
+ _target_: src.test.fashioniq.TestFashionIQ
18
+ category: shirt
configs/test/fashioniq-toptee.yaml ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ fashioniq-toptee:
2
+ dataname: fashioniq-toptee
3
+ _target_: src.data.fashioniq.FashionIQTestDataModule
4
+
5
+ batch_size: ${machine.batch_size}
6
+ num_workers: ${machine.num_workers}
7
+
8
+ annotation: ${paths.work_dir}/annotation/fashion-iq/cap.toptee.val.json
9
+ targets: ${paths.work_dir}/annotation/fashion-iq/split.toptee.val.json
10
+
11
+ img_dirs: ${paths.datasets_dir}/fashion-iq/images/
12
+ emb_dirs: ${paths.datasets_dir}/fashion-iq/blip-embs-large/
13
+
14
+ image_size: 384
15
+
16
+ test:
17
+ _target_: src.test.fashioniq.TestFashionIQ
18
+ category: toptee
configs/test/fashioniq.yaml ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ defaults:
2
+ - fashioniq-dress.yaml
3
+ - fashioniq-shirt.yaml
4
+ - fashioniq-toptee.yaml
configs/test/main.yaml ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ defaults:
2
+ - cirr.yaml
3
+ - webvid-covr.yaml
configs/test/webvid-covr.yaml ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ webvid_covr:
2
+ dataname: webvid-covr
3
+ _target_: src.data.webvid_covr.WebVidCoVRTestDataModule
4
+
5
+ image_size: 384
6
+
7
+ vid_query_method: middle
8
+ vid_frames: 1
9
+ emb_pool: query
10
+
11
+ batch_size: ${machine.batch_size}
12
+ num_workers: ${machine.num_workers}
13
+
14
+ # Paths
15
+ annotation: ${paths.work_dir}/annotation/webvid-covr/webvid8m-covr_test.csv
16
+ vid_dirs: ${paths.datasets_dir}/WebVid/8M/train
17
+ emb_dirs: ${paths.datasets_dir}/WebVid/8M/blip-vid-embs-${model.model.vit}-all
18
+
19
+ test:
20
+ _target_: src.test.webvid_covr.TestWebVidCoVR
configs/test/webvid-covr_text.yaml ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ webvid_covr_text:
2
+ dataname: webvid-covr_text
3
+ _target_: src.data.webvid_covr.WebVidCoVRTestDataModule
4
+
5
+ image_size: 384
6
+
7
+ vid_query_method: middle
8
+ vid_frames: 1
9
+ emb_pool: query
10
+
11
+ batch_size: ${machine.batch_size}
12
+ num_workers: ${machine.num_workers}
13
+
14
+ # Paths
15
+ annotation: ${paths.work_dir}/annotation/webvid-covr/webvid8m-covr_test.csv
16
+ vid_dirs: ${paths.datasets_dir}/WebVid/8M/train
17
+ emb_dirs: ${paths.datasets_dir}/WebVid/8M/blip-vid-embs-${model.model.vit}-all
18
+
19
+ test:
20
+ _target_: src.test.webvid_covr_exp.TestWebVidCoVRTextOnly