Update other_function.py

other_function.py

@@ -105,32 +105,39 @@ def get_weather(city):
   return (celcius)
 
 def get_rates():
+    chrome_options = Options()
+    chrome_options.add_argument("--headless")
+    chrome_options.add_argument("--no-sandbox")
+    chrome_options.add_argument("--disable-dev-shm-usage")
+    chrome_options.add_argument("--disable-gpu")
+    chrome_options.add_argument("--disable-extensions")
+    chrome_options.add_argument("--disable-infobars")
+    chrome_options.add_argument("--disable-images")
+    chrome_options.add_argument("--blink-settings=imagesEnabled=false")  # Disable image loading
+    chrome_options.add_argument("--enable-automation")
+
+    # Initialize the WebDriver with the headless option
+    driver = webdriver.Chrome(service=Service(), options=chrome_options)
+
     
-    proxies = {
-        "http": "http://160.86.242.23:8080",
-        "https": "http://160.86.242.23:8080",
-    }
+    # Open the website
+    driver.get('https://www.kisandeals.com/mandiprices/ALL/TAMIL-NADU/ALL')
     
-    r = requests.get('https://www.kisandeals.com/mandiprices/ALL/TAMIL-NADU/ALL', proxies=proxies)
-    soup = BeautifulSoup(r.text, 'html.parser')
+    # Wait for the table to be present (instead of waiting for the entire page)
+    table = WebDriverWait(driver, 5).until(
+        EC.presence_of_element_located((By.XPATH, '//table'))
+    )
     
-    # Finding the table in the HTML
-    table = soup.find('table')
+    # Parse the table using pandas
+    df = pd.read_html(table.get_attribute('outerHTML'))[0]
     
-    # Check if a table was found
-    if table:
-        # Convert the HTML table to a DataFrame
-        df = pd.read_html(str(table))[0]
-        
-        # Dropping the 'Quintal Price' column if it exists
-        if 'Quintal Price' in df.columns:
-            df.drop(columns=['Quintal Price'], inplace=True)
-        
-        # Creating a dictionary from the DataFrame
-        d = {}
-        for i in range(len(df)):
-            d[df.iloc[i, 0]] = df.iloc[i, 1]
-        
-        return str(d) + ' These prices are for 1 kg'
-    else:
-        return get_rates()
+    # Drop the 'Quintal Price' column
+    df.drop(columns=['Quintal Price'], inplace=True)
+    
+    # Convert the data to a dictionary
+    d = {df.iloc[i, 0]: df.iloc[i, 1] for i in range(len(df))}
+    
+    # Close the WebDriver
+    driver.quit()
+    
+    return str(d) + ' These prices are for 1 kg'