Update pipeline.py

pipeline.py

@@ -191,9 +191,44 @@ def download_drive_file_content(file_id):
 #             print(f"⏱️ Timeout exceeded ({timeout} sec) — function killed.")
 #             return False, None
 
+# import multiprocessing
+
+# def run_with_timeout(func, args=(), kwargs={}, timeout=30):
+#     def wrapper(q, *args, **kwargs):
+#         try:
+#             result = func(*args, **kwargs)
+#             q.put((True, result))
+#         except Exception as e:
+#             q.put((False, e))
+
+#     q = multiprocessing.Queue()
+#     p = multiprocessing.Process(target=wrapper, args=(q, *args), kwargs=kwargs)
+#     p.start()
+#     p.join(timeout)
+
+#     if p.is_alive():
+#         p.terminate()
+#         p.join()
+#         print(f"⏱️ Timeout exceeded ({timeout} sec) — function killed.")
+#         return False, None
+
+#     if not q.empty():
+#         success, result = q.get()
+#         if success:
+#             return True, result
+#         else:
+#             raise result  # re-raise exception if needed
+
+#     return False, None
+
 import multiprocessing
+import time
 
-def run_with_timeout(func, args=(), kwargs={}, timeout=30):
+def run_with_timeout(func, args=(), kwargs={}, timeout=30, stop_value=None):
+    """
+    Runs func in a separate process with optional timeout.
+    If stop_value is provided and becomes True during execution, the process is killed early.
+    """
     def wrapper(q, *args, **kwargs):
         try:
             result = func(*args, **kwargs)
@@ -204,24 +239,37 @@ def run_with_timeout(func, args=(), kwargs={}, timeout=30):
     q = multiprocessing.Queue()
     p = multiprocessing.Process(target=wrapper, args=(q, *args), kwargs=kwargs)
     p.start()
-    p.join(timeout)
 
-    if p.is_alive():
-        p.terminate()
-        p.join()
-        print(f"⏱️ Timeout exceeded ({timeout} sec) — function killed.")
-        return False, None
+    start_time = time.time()
+    while p.is_alive():
+        # Timeout check
+        if timeout is not None and (time.time() - start_time) > timeout:
+            p.terminate()
+            p.join()
+            print(f"⏱️ Timeout exceeded ({timeout} sec) — function killed.")
+            return False, None
 
+        # Stop flag check
+        if stop_value is not None and stop_value.value:
+            p.terminate()
+            p.join()
+            print("🛑 Stop flag detected — function killed early.")
+            return False, None
+
+        time.sleep(0.1)  # avoid busy waiting
+
+    # Process finished naturally
     if not q.empty():
         success, result = q.get()
         if success:
             return True, result
         else:
-            raise result  # re-raise exception if needed
+            raise result
 
     return False, None
 
 
+
 def time_it(func, *args, **kwargs):
     """
     Measure how long a function takes to run and return its result + time.