Update app.py

app.py

@@ -719,8 +719,8 @@ state = os.environ.get("state")
 system = os.environ.get("system")
 auth = os.environ.get("auth")
 auth2 = os.environ.get("auth2")
-openai.api_key = os.environ.get("auth")
-openai.api_base = os.environ.get("host")
+openai.api_key = auth
+openai.api_base = host
 vis_url = os.environ.get("vis_url")
 vis_auth = os.environ.get("vis_auth")
 endpoint = os.environ.get("endpoint")
@@ -783,7 +783,7 @@ def classify(platform, UserInput, Images, Textbox2, Textbox3):
                     return None
 
 
-            def vision():
+            def vision(caption):
                 # with open("image.png", "wb") as file1_write:
                 #     file1_write.write(image_data)
                 
@@ -798,38 +798,66 @@ def classify(platform, UserInput, Images, Textbox2, Textbox3):
                 if image_data:
                     try:
                         # Open the image directly from the image data
-                        image = Image.open(io.BytesIO(image_data))
-                        base64_image_str = encode_image(image)
+                        # image = Image.open(io.BytesIO(image_data))
+                        # base64_image_str = encode_image(image)
                         
-                        payload = {
-                            "content": [
-                                {
-                                    "prompt": "What's this image about? or What does this image contains?",
-                                    "image": base64_image_str,
-                                }
-                            ],
-                            "token": vis_auth,
-                        }
+                        # payload = {
+                        #     "content": [
+                        #         {
+                        #             "prompt": "What's this image about? or What does this image contains?",
+                        #             "image": base64_image_str,
+                        #         }
+                        #     ],
+                        #     "token": vis_auth,
+                        # }
                     
-                        url = vis_url
-                        headers = {"Content-Type": "application/json"}
+                        # url = vis_url
+                        # headers = {"Content-Type": "application/json"}
                     
-                        response = requests.post(url, headers=headers, data=json.dumps(payload))
-                        results = response.json()
-                        results = results["result"]
+                        # response = requests.post(url, headers=headers, data=json.dumps(payload))
+                        # results = response.json()
+                        # results = results["result"]
                     
-                        answer_index = results.find("Answer:")
+                        # answer_index = results.find("Answer:")
                     
-                        if answer_index != -1:
-                            try:
-                                result_text = results[answer_index + len("Answer:"):].strip()
-                                print(result_text)
-                                return result_text
-                            except:
-                                pass
-                        else:
-                            return "Answer: not found in the string."
+                        # if answer_index != -1:
+                        #     try:
+                        #         result_text = results[answer_index + len("Answer:"):].strip()
+                        #         print(result_text)
+                        #         return result_text
+                        #     except:
+                        #         pass
+                        # else:
+                        #     return "Answer: not found in the string."
+                        payload = json.dumps(
+                            {
+                                "messages": [
+                                    "role": "user",
+                                    "content": [
+                                        {
+                                            "type": "text",
+                                            "text": caption
+                                        },
+                            
+                                        {
+                                            "type": "image_url",
+                                            "image_url": Images
+                                        }
+                                    ]
+                                ],
+                                "model": "gemini-pro-vision"
+                            }
+                        )
+                        headers = {
+                            'Authorization': f'Bearer {auth}',
+                            'User-Agent': 'Apifox/1.0.0 (https://apifox.com)',
+                            'Content-Type': 'application/json'
+                        }
 
+                        response = requests.request("POST", host, headers=headers, data=payload)
+                        response = response.json()
+                        res = response["choices"][0]["message"]["content"]
+                        return res
                     except:
                         return "ERRRRRRR"
                 else:
@@ -838,6 +866,7 @@ def classify(platform, UserInput, Images, Textbox2, Textbox3):
 
             if UserInput is not None:
                 caption = UserInput.lower()
+                vision(caption)
             else:
                 caption = None