Update other_function.py

other_function.py

@@ -111,26 +111,46 @@ def get_weather(city):
   degree=temperature[:-2]
   celcius=str(round((int(degree) - 32)* 5/9,1))+temperature[-2]+'C'
   return (celcius)
-
 import scrapy
-from scrapy.selector import Selector
-from urllib.parse import urljoin
+from scrapy.crawler import CrawlerProcess
 import pandas as pd
 
-class KisanDealsSpider(scrapy.Spider):
-    name = 'kisan_deals'
+class RatesSpider(scrapy.Spider):
+    name = "rates_spider"
     start_urls = ['https://www.kisandeals.com/mandiprices/ALL/TAMIL-NADU/ALL']
 
     def parse(self, response):
-        table = response.xpath('//table')
-        rows = table.xpath('//tr')
+        # Extract the table data
+        table_rows = response.xpath('//table/tbody/tr')
 
+        # Initialize a list to hold the data
         data = []
-        for row in rows[1:]:  # Skip the header row
-            crop = row.xpath('//td[1]/text()').get()
-            rate = row.xpath('//td[2]/text()').get()
-            data.append({'crop': crop, 'rate': rate})
-
-        df = pd.DataFrame(data)
-        df = df.drop(columns=['Quintal Price'])
-        return str(df.to_dict('index')) + ' These prices are for 1 kg'
+
+        for row in table_rows:
+            # Extract the commodity name and price per kg
+            commodity_name = row.xpath('td[1]//text()').get().strip()
+            price_per_kg = row.xpath('td[2]//text()').get().strip()
+            
+            # Append the data to the list
+            data.append((commodity_name, price_per_kg))
+        
+        # Convert the data to a Pandas DataFrame
+        df = pd.DataFrame(data, columns=['Commodity', 'Price per kg'])
+        # Convert the DataFrame to a dictionary
+        rate_dict = df.set_index('Commodity')['Price per kg'].to_dict()
+
+        # Return the scraped rates
+        return rate_dict+' This is prices for 1 kg'
+
+def get_rates():
+    process = CrawlerProcess({
+        'USER_AGENT': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:89.0) Gecko/20100101 Firefox/89.0',
+        'LOG_LEVEL': 'ERROR',  # Suppress logging to keep the output clean
+    })
+
+    spider = RatesSpider()
+    process.crawl(spider)
+    process.start()  # This will block until the spider is done
+
+    # Get the scraped data from the spider
+    return spider.parse(None)