increase timeout for parallel

VerifiableRewardsForScalableLogicalReasoning.py

@@ -135,7 +135,7 @@ def _evaluate_with_prolog(prediction, validation_program, eval_config, timeout=5
     # extract predicate from rule_to_evaluate
     rule_to_evaluate = extract_ilp_from_text_v2(prediction, positive_pred, allow_multiple_rules)
     if positive_pred not in rule_to_evaluate:
-        logger.warning(f"Rule '{rule_to_evaluate}' does not contain positive predicate '{positive_pred}'")
+        logger.warning(f"Rule does not contain positive predicate '{positive_pred}'")
         return {
             "is_correct": False,
             "partial_score": 0.0,
@@ -200,7 +200,7 @@ check_all :- forall((pos({vars});neg({vars})), check({vars})).
 
         is_correct = True if partial_score == 1.0 else False
 
-        error = f'Rule invalid: "{rule_to_evaluate}" exit with ' + result.stderr if result.stderr else None
+        error =  f'{result.stderr} -> Eval Rule "{rule_to_evaluate}"' if result.stderr else None
         t1 = time.time()
 
         return {
@@ -212,7 +212,7 @@ check_all :- forall((pos({vars});neg({vars})), check({vars})).
         }
 
     except subprocess.TimeoutExpired:
-        logger.warning(f"Evaluation timed out after {timeout} seconds for rule: {rule_to_evaluate}...")
+        logger.warning(f"Evaluation timed out after {timeout} seconds for rule: '{rule_to_evaluate.replace('\n', ' ')}'")
         return {"is_correct": False, "partial_score": 0.0, "syntax_valid": False,
                 "error": f"Evaluation timed out after {timeout} seconds"}
     except Exception as e:
@@ -323,6 +323,7 @@ class VerifiableRewardsForScalableLogicalReasoning(evaluate.Metric):
             raise ValueError(
                 f"Number of predictions ({len(predictions)}) and references {len(references)}) don't match")
 
+        TIMEOUT = 15 if len(predictions) > 500 else 5
         # Prepare evaluation inputs
         eval_inputs = []
         for i, (prediction, reference) in enumerate(zip(predictions, references)):
@@ -338,7 +339,7 @@ class VerifiableRewardsForScalableLogicalReasoning(evaluate.Metric):
             if not validation_program:
                 raise ValueError(f"Example {i} does not contain validation program field")
 
-            eval_inputs.append((prediction, validation_program, eval_config))
+            eval_inputs.append((prediction, validation_program, eval_configm, TIMEOUT))
 
         # if more than 1k predictions, we use multiprocessing to speed up the evaluation
         if len(eval_inputs) > 500: