Update other_function.py

other_function.py

@@ -111,10 +111,8 @@ def get_weather(city):
   degree=temperature[:-2]
   celcius=str(round((int(degree) - 32)* 5/9,1))+temperature[-2]+'C'
   return (celcius)
-import scrapy
-from scrapy.crawler import CrawlerProcess
-import pandas as pd
-from scrapy.crawler import CrawlerProcess
+from twisted.internet import reactor, defer
+from scrapy.crawler import CrawlerRunner
 from scrapy.utils.project import get_project_settings
 import scrapy
 
@@ -129,22 +127,20 @@ class RateSpider(scrapy.Spider):
             commodity = row.xpath('td[1]/text()').get()
             price = row.xpath('td[2]/text()').get()
             data[commodity] = price
-        return data
+        yield data
 
+@defer.inlineCallbacks
 def get_rates():
-    # Set up a Scrapy process
-    process = CrawlerProcess(get_project_settings())
-    
-    # Set up a dictionary to store the scraped data
-    data = {}
+    data = []
 
-    # Run the spider
-    def crawler_finished(signal, sender, item, response, spider):
-        data.update(item)
-        
-    process.signals.connect(crawler_finished, signal=scrapy.signals.item_scraped)
-    process.crawl(RateSpider)
-    process.start()  # This will block until the crawling is finished
-
-    # Return the scraped data as a string (or format as needed)
-    return str(data) + ' These prices are for 1 kg'
+    def process_data(item, response, spider):
+        data.append(item)
+    
+    runner = CrawlerRunner(get_project_settings())
+    d = runner.crawl(RateSpider)
+    d.addBoth(lambda _: reactor.stop())
+    reactor.run()
+
+    # Convert the list of data to a string
+    result = ''.join([str(d) for d in data]) + ' These prices are for 1 kg'
+    return result