Fix: Bug 288 + Tests

openevolve/__init__.py

@@ -12,10 +12,11 @@
 )
 from openevolve.coevolution import CoevolutionResult, Opponent, Population, run_coevolution
 from openevolve.config import Config
-from openevolve.controller import OpenEvolve
+from openevolve.controller import EvolutionFailedError, OpenEvolve
 
 __all__ = [
     "Config",
+    "EvolutionFailedError",
     "OpenEvolve",
     "__version__",
     "run_evolution",

openevolve/cli.py

@@ -10,7 +10,7 @@
 import sys
 from typing import Any, Dict, List, Optional
 
-from openevolve import OpenEvolve
+from openevolve import EvolutionFailedError, OpenEvolve
 from openevolve.config import Config, load_config
 
 logger = logging.getLogger(__name__)
@@ -224,6 +224,11 @@ async def main_async() -> int:
 
         return 0
 
+    except EvolutionFailedError as e:
+        # Expected failure mode with an actionable message, no traceback needed
+        print(f"\n{e}")
+        return 1
+
     except Exception as e:
         print(f"Error: {str(e)}")
         import traceback

openevolve/controller.py

@@ -28,6 +28,11 @@
 logger = logging.getLogger(__name__)
 
 
+class EvolutionFailedError(RuntimeError):
+    """Raised when every iteration of a run failed without producing a candidate
+    program, e.g. because the LLM endpoint is unreachable (issue #288)"""
+
+
 class OpenEvolve:
     """
     Main controller for OpenEvolve
@@ -261,6 +266,9 @@ async def run(
 
         Returns:
             Best program found
+
+        Raises:
+            EvolutionFailedError: If iterations ran but none produced a candidate program
         """
         max_iterations = self.config.max_iterations if iterations is None else iterations
         # Determine starting iteration
@@ -375,6 +383,15 @@ def force_exit_handler(signum, frame):
                 evolution_start, evolution_iterations, target_score
             )
 
+            # A run where nothing was generated must not end looking like a run that
+            # simply found nothing better than the seed (issue #288)
+            if self.parallel_controller.produced_no_candidates:
+                raise EvolutionFailedError(
+                    f"Evolution failed: 0/{self.parallel_controller.completed_iterations} "
+                    f"iterations produced a candidate program. "
+                    f"Last error: {self.parallel_controller.last_iteration_error}"
+                )
+
         finally:
             # Clean up parallel processing resources
             if self.parallel_controller:

openevolve/process_parallel.py

@@ -487,6 +487,14 @@ def __init__(
         self.shutdown_event = mp.Event()
         self.early_stopping_triggered = False
 
+        # Per-run outcome counters, reset by run_evolution (issue #288)
+        self.completed_iterations = 0
+        self.successful_iterations = 0
+        self.last_iteration_error: Optional[str] = None
+        # True when the last run completed iterations but every one of them failed
+        # before producing a candidate program
+        self.produced_no_candidates = False
+
         # Number of worker processes
         self.num_workers = config.evaluator.parallel_evaluations
         self.num_islands = config.database.num_islands
@@ -691,6 +699,10 @@ async def run_evolution(
 
         next_iteration = current_iteration
         completed_iterations = 0
+        # Iterations whose worker produced a candidate program. A run where this
+        # stays at zero (e.g. the LLM endpoint is down) made no progress at all.
+        successful_iterations = 0
+        self.last_iteration_error = None
         total_prompt_tokens = 0
         total_completion_tokens = 0
         total_tokens = 0
@@ -752,10 +764,13 @@ async def run_evolution(
                     token_str = f" | tokens: {tt} (prompt: {pt}, completion: {ct})"
 
                 if result.error:
+                    self.last_iteration_error = result.error
                     logger.warning(
                         f"Iteration {completed_iteration} error: {result.error}{token_str}"
                     )
                 elif result.child_program_dict:
+                    successful_iterations += 1
+
                     # Reconstruct program from dict
                     child_program = Program(**result.child_program_dict)
 
@@ -971,6 +986,7 @@ async def run_evolution(
             except FutureTimeoutError as e:
                 if getattr(e, "_openevolve_strategy_error", False):
                     raise
+                self.last_iteration_error = f"Iteration timed out after {timeout_seconds}s"
                 logger.error(
                     f"⏰ Iteration {completed_iteration} timed out after {timeout_seconds}s "
                     f"(evaluator timeout: {self.config.evaluator.timeout}s + 30s buffer). "
@@ -981,6 +997,7 @@ async def run_evolution(
             except Exception as e:
                 if getattr(e, "_openevolve_strategy_error", False):
                     raise
+                self.last_iteration_error = str(e)
                 logger.error(f"Error processing result from iteration {completed_iteration}: {e}")
 
             completed_iterations += 1
@@ -1029,6 +1046,21 @@ async def run_evolution(
         else:
             logger.info("✅ Evolution completed - Maximum iterations reached")
 
+        self.completed_iterations = completed_iterations
+        self.successful_iterations = successful_iterations
+        # An interrupted run is not a failed one. Evaluated here, before stop()
+        # sets the shutdown event as part of normal cleanup.
+        self.produced_no_candidates = (
+            completed_iterations > 0
+            and successful_iterations == 0
+            and not self.shutdown_event.is_set()
+        )
+        if self.produced_no_candidates:
+            logger.error(
+                f"❌ None of the {completed_iterations} iterations produced a candidate program. "
+                f"Last error: {self.last_iteration_error}"
+            )
+
         if total_llm_calls > 0:
             logger.info(
                 f"📊 Total LLM Token Usage ({total_llm_calls} calls): "

tests/integration/README.md

@@ -50,6 +50,9 @@ All integration tests use:
 - **120 second timeout** per LLM call
 - **In-memory database** for speed
 - **Small iteration counts** (1-8 iterations) for CI compatibility
+- **Full rewrites** (`diff_based_evolution: false`): dhara-250m does not produce valid
+  SEARCH/REPLACE diffs, and a run where no iteration produces a candidate raises
+  `EvolutionFailedError` (#288), so the tests use full-rewrite mode to always get candidates
 
 ## CI Behavior
 

tests/integration/test_checkpoint_with_llm.py

@@ -88,9 +88,25 @@ async def test_checkpoint_resume_functionality(
                 latest_checkpoint = max(checkpoint_dirs, key=lambda d: int(d.name.split("_")[1]))
                 checkpoint_iter = int(latest_checkpoint.name.split("_")[1])
 
-                # Test resume (simplified - just verify the checkpoint directory structure)
-                assert (latest_checkpoint / "database.json").exists(), "Database checkpoint should exist"
+                # ProgramDatabase.save writes metadata.json plus one JSON file per program
+                assert (latest_checkpoint / "metadata.json").exists(), "Database metadata should exist"
+                saved_ids = {p.stem for p in (latest_checkpoint / "programs").glob("*.json")}
+                assert saved_ids, "Checkpoint should contain saved programs"
                 print(f"Successfully created checkpoint at iteration {checkpoint_iter}")
+
+                # Resume from the checkpoint and run a few more iterations
+                controller2 = OpenEvolve(
+                    initial_program_path=str(test_program_file),
+                    evaluation_file=str(test_evaluator_file),
+                    config=evolution_config,
+                    output_dir=str(evolution_output_dir)
+                )
+                await controller2.run(iterations=3, checkpoint_path=str(latest_checkpoint))
+
+                assert saved_ids <= set(controller2.database.programs), \
+                    "Resumed database should contain every program from the checkpoint"
+                assert controller2.database.last_iteration > checkpoint_iter, \
+                    "Resumed run should continue past the checkpoint iteration"
             else:
                 print("No checkpoints created (likely due to all iterations failing)")
         else:

tests/integration/test_library_api.py

@@ -36,6 +36,9 @@ def _get_library_test_config(port: int = 8000) -> Config:
     # Bound generation length so the (non-stopping) dhara model does not ramble to
     # the 4096-token default on every call (see tests/test_utils.py for details).
     config.llm.max_tokens = 256
+    # Full rewrites so the small model always yields a candidate (see
+    # get_integration_config in tests/test_utils.py); runs with no candidates fail.
+    config.diff_based_evolution = False
     config.llm.models = [
         LLMModelConfig(
             name=TEST_MODEL,
@@ -74,7 +77,7 @@ def simple_multiply(x, y):
         result = evolve_function(
             simple_multiply,
             test_cases,
-            iterations=2,  # Very small number for CI speed
+            iterations=3,  # A few iterations: one empty reply from the small model would leave no candidate
             output_dir=str(temp_workspace / "evolve_function_output"),
             cleanup=False,  # Keep files for inspection
             config=_get_library_test_config(optillm_server["port"]),
@@ -162,7 +165,7 @@ def fibonacci_evaluator(program_path):
         result = evolve_code(
             initial_code,
             fibonacci_evaluator,
-            iterations=1,  # Minimal for CI speed
+            iterations=3,  # A few iterations: one empty reply from the small model would leave no candidate
             output_dir=str(temp_workspace / "evolve_code_output"),
             cleanup=False,  # Keep output directory
             config=_get_library_test_config(optillm_server["port"]),
@@ -257,7 +260,7 @@ def evaluate(program_path):
         result = run_evolution(
             initial_program=str(initial_program),
             evaluator=str(evaluator_file),
-            iterations=1,  # Minimal for CI speed
+            iterations=3,  # A few iterations: one empty reply from the small model would leave no candidate
             output_dir=str(temp_workspace / "run_evolution_output"),
             cleanup=False,  # Keep output directory
             config=_get_library_test_config(optillm_server["port"]),
@@ -284,7 +287,7 @@ def evaluate(program_path):
         result2 = run_evolution(
             initial_program=initial_program.read_text(),
             evaluator=lambda path: {"score": 0.8, "test": "passed"},  # Simple callable evaluator
-            iterations=1,
+            iterations=3,  # A few iterations: one empty reply from the small model would leave no candidate
             output_dir=str(temp_workspace / "run_evolution_string_output"),
             cleanup=False,  # Keep output directory
             config=_get_library_test_config(optillm_server["port"]),