Update process status and config

yourbench_space/app.py

@@ -60,6 +60,16 @@ def generate_and_return(hf_org, hf_prefix):
         )
     )
 
+def update_process_status():
+    """Update process status and include exit details if process has terminated"""
+    is_running = manager.is_running()
+    
+    if not is_running:
+        exit_code, exit_reason = manager.get_exit_details()
+        status_text = f"Process Status: Stopped - {exit_reason}, exit code - {exit_code}" if exit_reason else "Process Status: Stopped"
+        return gr.update(value=False, label=status_text)
+    
+    return gr.update(value=True, label="Process Status: Running")
 
 def prepare_task(oauth_token: gr.OAuthToken | None, hf_dataset_prefix: str, _=None):
     new_env = os.environ.copy()
@@ -121,7 +131,7 @@ with gr.Blocks(theme=gr.themes.Default()) as app:
                     file_input = gr.File(
                         label="Upload text files",
                         file_count="multiple",
-                        file_types=[".txt", ".md", ".html"],
+                        file_types=[".txt", ".md", ".html", ".pdf"],
                     )
                     output = gr.Textbox(label="Log")
                     file_input.upload(
@@ -170,7 +180,7 @@ with gr.Blocks(theme=gr.themes.Default()) as app:
             with gr.Row():
                 process_status = gr.Checkbox(label="Process Status", interactive=False)
                 status_timer = gr.Timer(1.0, active=True)
-                status_timer.tick(manager.is_running, outputs=process_status)
+                status_timer.tick(update_process_status, outputs=process_status)
 
             with gr.Row():
                 start_button = gr.Button("Start Task")

yourbench_space/config.py

@@ -12,16 +12,15 @@ def generate_base_config(hf_org, hf_prefix):
             "hf_organization": hf_org,
             "hf_dataset_name": hf_prefix,
         },
-        "local_dataset_dir": "results/",
         "model_list": [
             {
                 "model_name": "meta-llama/Llama-3.3-70B-Instruct",
-                "provider": "sambanova",
+                "provider": "novita",
                 "max_concurrent_requests": 32,
             },
             {
                 "model_name": "Qwen/Qwen2.5-72B-Instruct",
-                "provider": "nebius",
+                "provider": "novita",
                 "max_concurrent_requests": 32,
             }
         ],
@@ -76,7 +75,7 @@ def generate_base_config(hf_org, hf_prefix):
                 ],
             },
             "judge_answers": {
-                "run": True,
+                "run": False, # to change when fixed
                 "comparing_strategies": [["zeroshot", "gold"]],
                 "chunk_column_index": 0,
                 "random_seed": 42,

yourbench_space/utils.py

@@ -19,9 +19,8 @@ STAGES = [
     "summarization",
     "chunking",
     "single_shot_question_generation",
-    "multi_hop_question_generation",
     "answer_generation",
-    "judge_answers",
+    # "judge_answers", # to uncomment when fixed 
 ]
 
 
@@ -56,6 +55,7 @@ class SubprocessManager:
         self.command = command
         self.process = None
         self.output_stream = io.StringIO()
+        self.exit_code = None
 
     def start_process(self, custom_env: dict | None):
         """Start the subprocess."""
@@ -64,20 +64,30 @@ class SubprocessManager:
             return
 
         self.output_stream = io.StringIO()
-        self.process = subprocess.Popen(
-            self.command,
-            stdout=subprocess.PIPE,
-            stderr=subprocess.STDOUT,  # Combine stderr with stdout
-            text=True,
-            bufsize=1,
-            start_new_session=True,
-            env=custom_env,
-        )
-        os.set_blocking(self.process.stdout.fileno(), False)
-        logger.info("Started the process")
+        self.exit_code = None
+        
+        try:
+            logger.info(f"Starting process with command: {' '.join(self.command)}")
+            self.process = subprocess.Popen(
+                self.command,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT,  # Combine stderr with stdout
+                text=True,
+                bufsize=1,
+                start_new_session=True,
+                env=custom_env,
+            )
+            os.set_blocking(self.process.stdout.fileno(), False)
+            logger.info(f"Started process with PID: {self.process.pid}")
+        except Exception as e:
+            logger.error(f"Failed to start process: {str(e)}")
+            return
 
     def read_and_get_output(self):
         """Read subprocess output, capture it, and return log and completed stages."""
+        current_output = ""
+        completed_stages = []
+
         if self.process and self.process.stdout:
             try:
                 while True:
@@ -89,10 +99,9 @@ class SubprocessManager:
             except BlockingIOError:
                 pass
 
-        current_output = self.output_stream.getvalue()
-        completed_stages = list(
-            set(re.findall(r"Successfully completed stage: (\w+)", current_output))
-        )
+            current_output = self.output_stream.getvalue()
+            completed_stages = list(set(re.findall(r"Successfully completed stage: (\w+)", current_output)))
+
         return current_output, completed_stages
 
     def stop_process(self):
@@ -101,10 +110,13 @@ class SubprocessManager:
             logger.info("Process is not running")
             return
         logger.info("Sending SIGTERM to the Process")
-        self.process.terminate()
-        exit_code = self.process.wait()  # Wait for process to terminate
-        logger.info(f"Process stopped exit code {exit_code}")
-        # return exit_code
+        try:
+            self.process.terminate()
+            self.exit_code =  self.process.wait(timeout=5)  # Wait up to 5 seconds for process to terminate
+            logger.info(f"Process terminated by user with exit code {self.exit_code}")
+        except subprocess.TimeoutExpired:
+            logger.warning("Process did not terminate within timeout, sending SIGKILL")
+            self.kill_process()
 
     def kill_process(self):
         """Forcefully kill the subprocess"""
@@ -112,11 +124,29 @@ class SubprocessManager:
             logger.info("Process is not running")
             return
         logger.info("Sending SIGKILL to the Process")
-        self.process.kill()
-        exit_code = self.process.wait()  # Wait for process to be killed
-        logger.info(f"Process killed exit code {exit_code}")
-        # return exit_code
+        try:
+            self.process.kill()
+            self.exit_code = self.process.wait(timeout=5) # Wait up to 5 seconds for process to be killed
+            logger.info(f"Process killed by user with exit code {self.exit_code}")
+        except subprocess.TimeoutExpired:
+            logger.error("Process could not be killed within timeout")
 
     def is_running(self):
         """Check if the subprocess is still running"""
-        return self.process and self.process.poll() is None
+        if self.process is None:
+            return False
+
+        return self.process.poll() is None
+
+    def get_exit_details(self):
+        """Return exit code and reason if process has terminated"""
+        if self.process is None:
+            return None, "Process was never started"
+        
+        if self.is_running():
+            return None, "Process is still running"
+        
+        if not self.exit_code is None and self.exit_code != 0 :
+            return self.exit_code, "Process exited abnormaly"
+
+        return self.exit_code, "Process exited normaly"