dags/CrawDag/sending/SendingTask.py

from CrawDag.models import TaskHandle, DataExchange, News
from CrawDag.saving import SavingTask
from CrawDag.saving.SavingMethod import MongoDataLake
import time
import requests
from dotenv import load_dotenv
import os

load_dotenv()

class SendingTask(TaskHandle):
    task_ids = None
    key = 'send_news'

    def __init__(self, task_ids: str) -> None:
        super().__init__()
        SendingTask.task_ids = task_ids
        self.__maxRetry = 4
        self.__delay = 15

    def execute(self, **context: any):
        dataExchange = DataExchange(context['ti'])
        listNewsId = dataExchange.pull(SavingTask.task_ids, SavingTask.key)

        for attempt in range(self.__maxRetry):
            try:
                response = requests.post(
                    url=os.getenv("SERVER_URL") + '/api/news/summarize',
                    json=listNewsId
                )
                if response.status_code == 202:
                    return
                else:
                    print(f"Attempt {attempt + 1} failed: {response.status_code}")
                    time.sleep(self.__delay)

            except requests.RequestException as e: 
                print(f"Attempt {attempt + 1} failed: {e}")
            time.sleep(self.__delay)
        
        dataLake = SavingTask(SavingTask.task_ids).dataLake
        dataLake.delete(listNewsId)
        print('Failed to send data to API')
        raise Exception('Failed to send data to API')