Update app.py

app.py

@@ -96,38 +96,38 @@ Return the response in the following JSON response format:
         "max_tokens": 4096
     }
 
-    try:
-        response = requests.post("https://api.openai.com/v1/chat/completions", headers=headers, json=payload)
-        logging.info(f"Full API response: {response.text}")
-        if response.status_code == 200:
-            json_response = response.json()
-            response_content = json_response["choices"][0]["message"]["content"]
-            if response_content:
-                try:
-                    json_string = response_content[response_content.find("{"): response_content.rfind("}") + 1]
-                    json_data = json.loads(json_string)
-                    if "data" in json_data and json_data["data"]:
-                        return json_data["data"]
-                    else:
-                        logging.error(f"No records found in page/file: {page_identifier}")
-                        error_pages.append(page_identifier)
-                        return []
-                except json.JSONDecodeError:
-                    logging.error(f"JSON parsing error in response for page/file {page_identifier}")
+   try:
+    response = requests.post("https://api.openai.com/v1/chat/completions", headers=headers, json=payload)
+    logging.info(f"Full API response: {response.text}")
+    if response.status_code == 200:
+        response_text = response.text
+        # Use regex to extract JSON from the API response content
+        matches = re.findall(r"```json\n(\{.*?\})\n```", response_text, re.DOTALL)
+        if matches:
+            try:
+                json_data = json.loads(matches[0])  # Assuming the first match is the required JSON data
+                if "data" in json_data and json_data["data"]:
+                    return json_data["data"]
+                else:
+                    logging.error(f"No records found in page/file: {page_identifier}")
                     error_pages.append(page_identifier)
                     return []
-            else:
-                logging.error(f"No content in JSON response for page/file {page_identifier}")
+            except json.JSONDecodeError:
+                logging.error(f"JSON parsing error in response for page/file {page_identifier}")
                 error_pages.append(page_identifier)
                 return []
         else:
-            logging.error(f"Error in API call for page/file {page_identifier}: HTTP {response.status_code} - {response.text}")
+            logging.error(f"No valid JSON found in response for page/file {page_identifier}")
             error_pages.append(page_identifier)
             return []
-    except requests.exceptions.RequestException as e:
-        logging.error(f"Network or API error when processing page/file {page_identifier}: {e}")
+    else:
+        logging.error(f"Error in API call for page/file {page_identifier}: HTTP {response.status_code} - {response.text}")
         error_pages.append(page_identifier)
         return []
+except requests.exceptions.RequestException as e:
+    logging.error(f"Network or API error when processing page/file {page_identifier}: {e}")
+    error_pages.append(page_identifier)
+    return []
 
 def process_pdf_and_generate_csv(file_path):
     error_pages = []  # Initialize the list to track error pages or files