diff --git a/openevolve/__init__.py b/openevolve/__init__.py index 32972f5f0..947b2fa4f 100644 --- a/openevolve/__init__.py +++ b/openevolve/__init__.py @@ -12,10 +12,11 @@ ) from openevolve.coevolution import CoevolutionResult, Opponent, Population, run_coevolution from openevolve.config import Config -from openevolve.controller import OpenEvolve +from openevolve.controller import EvolutionFailedError, OpenEvolve __all__ = [ "Config", + "EvolutionFailedError", "OpenEvolve", "__version__", "run_evolution", diff --git a/openevolve/cli.py b/openevolve/cli.py index a0e5bd1e1..26d5275be 100644 --- a/openevolve/cli.py +++ b/openevolve/cli.py @@ -10,7 +10,7 @@ import sys from typing import Any, Dict, List, Optional -from openevolve import OpenEvolve +from openevolve import EvolutionFailedError, OpenEvolve from openevolve.config import Config, load_config logger = logging.getLogger(__name__) @@ -224,6 +224,11 @@ async def main_async() -> int: return 0 + except EvolutionFailedError as e: + # Expected failure mode with an actionable message, no traceback needed + print(f"\n{e}") + return 1 + except Exception as e: print(f"Error: {str(e)}") import traceback diff --git a/openevolve/controller.py b/openevolve/controller.py index 6b6a00684..c0e5d386d 100644 --- a/openevolve/controller.py +++ b/openevolve/controller.py @@ -28,6 +28,11 @@ logger = logging.getLogger(__name__) +class EvolutionFailedError(RuntimeError): + """Raised when every iteration of a run failed without producing a candidate + program, e.g. because the LLM endpoint is unreachable (issue #288)""" + + class OpenEvolve: """ Main controller for OpenEvolve @@ -261,6 +266,9 @@ async def run( Returns: Best program found + + Raises: + EvolutionFailedError: If iterations ran but none produced a candidate program """ max_iterations = self.config.max_iterations if iterations is None else iterations # Determine starting iteration @@ -375,6 +383,15 @@ def force_exit_handler(signum, frame): evolution_start, evolution_iterations, target_score ) + # A run where nothing was generated must not end looking like a run that + # simply found nothing better than the seed (issue #288) + if self.parallel_controller.produced_no_candidates: + raise EvolutionFailedError( + f"Evolution failed: 0/{self.parallel_controller.completed_iterations} " + f"iterations produced a candidate program. " + f"Last error: {self.parallel_controller.last_iteration_error}" + ) + finally: # Clean up parallel processing resources if self.parallel_controller: diff --git a/openevolve/process_parallel.py b/openevolve/process_parallel.py index 8c2b29adf..33c19303e 100644 --- a/openevolve/process_parallel.py +++ b/openevolve/process_parallel.py @@ -487,6 +487,14 @@ def __init__( self.shutdown_event = mp.Event() self.early_stopping_triggered = False + # Per-run outcome counters, reset by run_evolution (issue #288) + self.completed_iterations = 0 + self.successful_iterations = 0 + self.last_iteration_error: Optional[str] = None + # True when the last run completed iterations but every one of them failed + # before producing a candidate program + self.produced_no_candidates = False + # Number of worker processes self.num_workers = config.evaluator.parallel_evaluations self.num_islands = config.database.num_islands @@ -691,6 +699,10 @@ async def run_evolution( next_iteration = current_iteration completed_iterations = 0 + # Iterations whose worker produced a candidate program. A run where this + # stays at zero (e.g. the LLM endpoint is down) made no progress at all. + successful_iterations = 0 + self.last_iteration_error = None total_prompt_tokens = 0 total_completion_tokens = 0 total_tokens = 0 @@ -752,10 +764,13 @@ async def run_evolution( token_str = f" | tokens: {tt} (prompt: {pt}, completion: {ct})" if result.error: + self.last_iteration_error = result.error logger.warning( f"Iteration {completed_iteration} error: {result.error}{token_str}" ) elif result.child_program_dict: + successful_iterations += 1 + # Reconstruct program from dict child_program = Program(**result.child_program_dict) @@ -971,6 +986,7 @@ async def run_evolution( except FutureTimeoutError as e: if getattr(e, "_openevolve_strategy_error", False): raise + self.last_iteration_error = f"Iteration timed out after {timeout_seconds}s" logger.error( f"⏰ Iteration {completed_iteration} timed out after {timeout_seconds}s " f"(evaluator timeout: {self.config.evaluator.timeout}s + 30s buffer). " @@ -981,6 +997,7 @@ async def run_evolution( except Exception as e: if getattr(e, "_openevolve_strategy_error", False): raise + self.last_iteration_error = str(e) logger.error(f"Error processing result from iteration {completed_iteration}: {e}") completed_iterations += 1 @@ -1029,6 +1046,21 @@ async def run_evolution( else: logger.info("✅ Evolution completed - Maximum iterations reached") + self.completed_iterations = completed_iterations + self.successful_iterations = successful_iterations + # An interrupted run is not a failed one. Evaluated here, before stop() + # sets the shutdown event as part of normal cleanup. + self.produced_no_candidates = ( + completed_iterations > 0 + and successful_iterations == 0 + and not self.shutdown_event.is_set() + ) + if self.produced_no_candidates: + logger.error( + f"❌ None of the {completed_iterations} iterations produced a candidate program. " + f"Last error: {self.last_iteration_error}" + ) + if total_llm_calls > 0: logger.info( f"📊 Total LLM Token Usage ({total_llm_calls} calls): " diff --git a/tests/integration/README.md b/tests/integration/README.md index a36306488..c933d808a 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -50,6 +50,9 @@ All integration tests use: - **120 second timeout** per LLM call - **In-memory database** for speed - **Small iteration counts** (1-8 iterations) for CI compatibility +- **Full rewrites** (`diff_based_evolution: false`): dhara-250m does not produce valid + SEARCH/REPLACE diffs, and a run where no iteration produces a candidate raises + `EvolutionFailedError` (#288), so the tests use full-rewrite mode to always get candidates ## CI Behavior diff --git a/tests/integration/test_checkpoint_with_llm.py b/tests/integration/test_checkpoint_with_llm.py index e801f9cfb..492fab3fc 100644 --- a/tests/integration/test_checkpoint_with_llm.py +++ b/tests/integration/test_checkpoint_with_llm.py @@ -88,9 +88,25 @@ async def test_checkpoint_resume_functionality( latest_checkpoint = max(checkpoint_dirs, key=lambda d: int(d.name.split("_")[1])) checkpoint_iter = int(latest_checkpoint.name.split("_")[1]) - # Test resume (simplified - just verify the checkpoint directory structure) - assert (latest_checkpoint / "database.json").exists(), "Database checkpoint should exist" + # ProgramDatabase.save writes metadata.json plus one JSON file per program + assert (latest_checkpoint / "metadata.json").exists(), "Database metadata should exist" + saved_ids = {p.stem for p in (latest_checkpoint / "programs").glob("*.json")} + assert saved_ids, "Checkpoint should contain saved programs" print(f"Successfully created checkpoint at iteration {checkpoint_iter}") + + # Resume from the checkpoint and run a few more iterations + controller2 = OpenEvolve( + initial_program_path=str(test_program_file), + evaluation_file=str(test_evaluator_file), + config=evolution_config, + output_dir=str(evolution_output_dir) + ) + await controller2.run(iterations=3, checkpoint_path=str(latest_checkpoint)) + + assert saved_ids <= set(controller2.database.programs), \ + "Resumed database should contain every program from the checkpoint" + assert controller2.database.last_iteration > checkpoint_iter, \ + "Resumed run should continue past the checkpoint iteration" else: print("No checkpoints created (likely due to all iterations failing)") else: diff --git a/tests/integration/test_library_api.py b/tests/integration/test_library_api.py index d050d07bb..5633536a2 100644 --- a/tests/integration/test_library_api.py +++ b/tests/integration/test_library_api.py @@ -36,6 +36,9 @@ def _get_library_test_config(port: int = 8000) -> Config: # Bound generation length so the (non-stopping) dhara model does not ramble to # the 4096-token default on every call (see tests/test_utils.py for details). config.llm.max_tokens = 256 + # Full rewrites so the small model always yields a candidate (see + # get_integration_config in tests/test_utils.py); runs with no candidates fail. + config.diff_based_evolution = False config.llm.models = [ LLMModelConfig( name=TEST_MODEL, @@ -74,7 +77,7 @@ def simple_multiply(x, y): result = evolve_function( simple_multiply, test_cases, - iterations=2, # Very small number for CI speed + iterations=3, # A few iterations: one empty reply from the small model would leave no candidate output_dir=str(temp_workspace / "evolve_function_output"), cleanup=False, # Keep files for inspection config=_get_library_test_config(optillm_server["port"]), @@ -162,7 +165,7 @@ def fibonacci_evaluator(program_path): result = evolve_code( initial_code, fibonacci_evaluator, - iterations=1, # Minimal for CI speed + iterations=3, # A few iterations: one empty reply from the small model would leave no candidate output_dir=str(temp_workspace / "evolve_code_output"), cleanup=False, # Keep output directory config=_get_library_test_config(optillm_server["port"]), @@ -257,7 +260,7 @@ def evaluate(program_path): result = run_evolution( initial_program=str(initial_program), evaluator=str(evaluator_file), - iterations=1, # Minimal for CI speed + iterations=3, # A few iterations: one empty reply from the small model would leave no candidate output_dir=str(temp_workspace / "run_evolution_output"), cleanup=False, # Keep output directory config=_get_library_test_config(optillm_server["port"]), @@ -284,7 +287,7 @@ def evaluate(program_path): result2 = run_evolution( initial_program=initial_program.read_text(), evaluator=lambda path: {"score": 0.8, "test": "passed"}, # Simple callable evaluator - iterations=1, + iterations=3, # A few iterations: one empty reply from the small model would leave no candidate output_dir=str(temp_workspace / "run_evolution_string_output"), cleanup=False, # Keep output directory config=_get_library_test_config(optillm_server["port"]), diff --git a/tests/test_all_iterations_failed.py b/tests/test_all_iterations_failed.py new file mode 100644 index 000000000..136f88c52 --- /dev/null +++ b/tests/test_all_iterations_failed.py @@ -0,0 +1,272 @@ +""" +Tests for runs where every iteration fails (issue #288) + +When the LLM endpoint is down, every iteration errors out before producing a +candidate program. Such a run used to finish with "Evolution complete!", exit +code 0 and the untouched seed saved as the best program, which is +indistinguishable from a run that simply found nothing better. +""" + +import asyncio +import http.server +import json +import os +import shutil +import subprocess +import sys +import tempfile +import threading +import unittest +from unittest.mock import MagicMock, patch + +# Set dummy API key for testing +os.environ["OPENAI_API_KEY"] = "test" + +from openevolve import EvolutionFailedError, OpenEvolve +from openevolve.config import Config +from openevolve.database import Program, ProgramDatabase +from openevolve.process_parallel import ProcessParallelController, SerializableResult + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +EVALUATOR_CODE = """ +def evaluate(program_path): + return {"combined_score": 0.5} +""" + + +def _error_future(iteration, error="LLM generation failed: Connection error."): + future = MagicMock() + future.done.return_value = True + future.result.return_value = SerializableResult(error=error, iteration=iteration) + return future + + +def _child_future(iteration, island_id): + future = MagicMock() + future.done.return_value = True + future.result.return_value = SerializableResult( + child_program_dict={ + "id": f"child_{iteration}", + "code": f"def evolved_{iteration}(): return {iteration}", + "language": "python", + "parent_id": "seed", + "generation": 1, + "metrics": {"combined_score": 0.4}, + "iteration_found": iteration, + "metadata": {"changes": "test", "island": island_id}, + }, + parent_id="seed", + iteration_time=0.1, + iteration=iteration, + ) + return future + + +class TestControllerCountsCandidates(unittest.TestCase): + """ProcessParallelController must report whether any iteration produced a candidate""" + + def setUp(self): + self.test_dir = tempfile.mkdtemp() + self.eval_file = os.path.join(self.test_dir, "evaluator.py") + with open(self.eval_file, "w") as f: + f.write(EVALUATOR_CODE) + + self.config = Config() + self.config.evaluator.parallel_evaluations = 2 + self.config.evaluator.timeout = 10 + self.config.database.num_islands = 1 + self.config.database.in_memory = True + + self.database = ProgramDatabase(self.config.database) + self.database.add( + Program( + id="seed", + code="def seed(): return 0", + language="python", + metrics={"combined_score": 0.5}, + iteration_found=0, + ) + ) + + def tearDown(self): + shutil.rmtree(self.test_dir, ignore_errors=True) + + def _run(self, make_future, max_iterations=4, before_run=None): + async def run_test(): + controller = ProcessParallelController(self.config, self.eval_file, self.database) + with patch.object(controller, "_submit_iteration", side_effect=make_future): + controller.start() + try: + if before_run: + before_run(controller) + await controller.run_evolution( + start_iteration=1, max_iterations=max_iterations, target_score=None + ) + finally: + controller.stop() + return controller + + return asyncio.run(run_test()) + + def test_all_errors_produce_no_candidates(self): + controller = self._run(lambda i, island: _error_future(i)) + + self.assertTrue(controller.produced_no_candidates) + self.assertEqual(controller.completed_iterations, 4) + self.assertEqual(controller.successful_iterations, 0) + self.assertEqual( + controller.last_iteration_error, "LLM generation failed: Connection error." + ) + + def test_single_success_counts_as_progress(self): + """Intermittent failures are still tolerated as long as something was generated""" + controller = self._run( + lambda i, island: _child_future(i, island) if i == 3 else _error_future(i) + ) + + self.assertFalse(controller.produced_no_candidates) + self.assertEqual(controller.successful_iterations, 1) + + def test_shutdown_is_not_a_failure(self): + """An interrupted run is not reported as a failed one""" + controller = self._run( + lambda i, island: _error_future(i), + before_run=lambda c: c.request_shutdown(), + ) + + self.assertFalse(controller.produced_no_candidates) + + +class TestOpenEvolveRunFailsWithoutCandidates(unittest.TestCase): + """OpenEvolve.run must raise instead of presenting the seed as the evolved best""" + + def setUp(self): + self.test_dir = tempfile.mkdtemp() + self.program_file = os.path.join(self.test_dir, "initial_program.py") + with open(self.program_file, "w") as f: + f.write("# EVOLVE-BLOCK-START\ndef f():\n return 0\n# EVOLVE-BLOCK-END\n") + self.eval_file = os.path.join(self.test_dir, "evaluator.py") + with open(self.eval_file, "w") as f: + f.write(EVALUATOR_CODE) + self.output_dir = os.path.join(self.test_dir, "out") + + self.config = Config() + self.config.evaluator.parallel_evaluations = 1 + self.config.database.num_islands = 1 + self.config.database.in_memory = True + + def tearDown(self): + shutil.rmtree(self.test_dir, ignore_errors=True) + + def _run(self, make_future): + openevolve = OpenEvolve( + initial_program_path=self.program_file, + evaluation_file=self.eval_file, + config=self.config, + output_dir=self.output_dir, + ) + with patch.object(ProcessParallelController, "_submit_iteration", autospec=True) as m: + m.side_effect = lambda _self, i, island=None: make_future(i, island) + return asyncio.run(openevolve.run(iterations=3)) + + def test_raises_and_does_not_save_seed_as_best(self): + with self.assertRaises(EvolutionFailedError) as ctx: + self._run(lambda i, island: _error_future(i)) + + self.assertIn("0/3 iterations produced a candidate program", str(ctx.exception)) + self.assertIn("Connection error", str(ctx.exception)) + self.assertFalse(os.path.exists(os.path.join(self.output_dir, "best"))) + + def test_partial_success_still_returns_best(self): + best = self._run(lambda i, island: _child_future(i, island) if i == 2 else _error_future(i)) + + self.assertIsNotNone(best) + self.assertTrue(os.path.exists(os.path.join(self.output_dir, "best"))) + + +class _DeadEndpointHandler(http.server.BaseHTTPRequestHandler): + """Answers every LLM request with HTTP 404, like a missing deployment""" + + def do_POST(self): + self.rfile.read(int(self.headers.get("Content-Length", 0))) + body = json.dumps( + {"error": {"code": "DeploymentNotFound", "message": "deployment does not exist"}} + ).encode() + self.send_response(404) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args): + pass + + +class TestCLIExitCodeWithDeadEndpoint(unittest.TestCase): + """End-to-end CLI check against a dead LLM endpoint (reproduction from issue #288)""" + + def setUp(self): + self.server = http.server.HTTPServer(("127.0.0.1", 0), _DeadEndpointHandler) + threading.Thread(target=self.server.serve_forever, daemon=True).start() + self.test_dir = tempfile.mkdtemp() + + def tearDown(self): + self.server.shutdown() + self.server.server_close() + shutil.rmtree(self.test_dir, ignore_errors=True) + + def test_exits_non_zero_without_success_banner(self): + program_file = os.path.join(self.test_dir, "initial_program.py") + with open(program_file, "w") as f: + f.write("# EVOLVE-BLOCK-START\ndef f():\n return 0\n# EVOLVE-BLOCK-END\n") + eval_file = os.path.join(self.test_dir, "evaluator.py") + with open(eval_file, "w") as f: + f.write(EVALUATOR_CODE) + config_file = os.path.join(self.test_dir, "config.yaml") + with open(config_file, "w") as f: + f.write( + "llm:\n" + f" api_base: http://127.0.0.1:{self.server.server_port}/v1\n" + " primary_model: dead-model\n" + " retries: 0\n" + " retry_delay: 0\n" + " timeout: 5\n" + "database:\n" + " num_islands: 1\n" + "evaluator:\n" + " parallel_evaluations: 1\n" + ) + output_dir = os.path.join(self.test_dir, "out") + + env = dict(os.environ) + env.setdefault("OPENAI_API_KEY", "test-key-for-unit-tests") + result = subprocess.run( + [ + sys.executable, + "-m", + "openevolve.cli", + program_file, + eval_file, + "--config", + config_file, + "--iterations", + "2", + "--output", + output_dir, + ], + cwd=REPO_ROOT, + env=env, + capture_output=True, + text=True, + timeout=120, + ) + + self.assertEqual(result.returncode, 1, result.stdout + result.stderr) + self.assertIn("0/2 iterations produced a candidate program", result.stdout) + self.assertNotIn("Evolution complete!", result.stdout) + self.assertFalse(os.path.exists(os.path.join(output_dir, "best"))) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_checkpoint_resume.py b/tests/test_checkpoint_resume.py index 791d5d5d8..5d2e5662e 100644 --- a/tests/test_checkpoint_resume.py +++ b/tests/test_checkpoint_resume.py @@ -114,6 +114,7 @@ async def run_test(): mock_controller.stop = Mock(return_value=None) mock_controller.shutdown_event = Mock() mock_controller.shutdown_event.is_set.return_value = False + mock_controller.produced_no_candidates = False mock_controller_class.return_value = mock_controller # Run for 0 iterations (just initialization) @@ -170,6 +171,7 @@ async def run_test(): mock_controller.stop = Mock(return_value=None) mock_controller.shutdown_event = Mock() mock_controller.shutdown_event.is_set.return_value = False + mock_controller.produced_no_candidates = False mock_controller_class.return_value = mock_controller # Run for 0 iterations (just initialization) @@ -225,6 +227,7 @@ async def run_test(): mock_controller.stop = Mock(return_value=None) mock_controller.shutdown_event = Mock() mock_controller.shutdown_event.is_set.return_value = False + mock_controller.produced_no_candidates = False mock_controller_class.return_value = mock_controller # Run for 0 iterations (just initialization) @@ -283,6 +286,7 @@ async def run_test(): mock_controller.stop = Mock(return_value=None) mock_controller.shutdown_event = Mock() mock_controller.shutdown_event.is_set.return_value = False + mock_controller.produced_no_candidates = False mock_controller_class.return_value = mock_controller # Run for 0 iterations (just initialization) @@ -324,6 +328,7 @@ async def run_test(): mock_parallel.start = MagicMock() mock_parallel.stop = MagicMock() mock_parallel.shutdown_event.is_set.return_value = False + mock_parallel.produced_no_candidates = False mock_parallel_class.return_value = mock_parallel # Run first time diff --git a/tests/test_initial_program_artifacts.py b/tests/test_initial_program_artifacts.py index 079ff9eea..dcf4e47d7 100644 --- a/tests/test_initial_program_artifacts.py +++ b/tests/test_initial_program_artifacts.py @@ -84,6 +84,7 @@ def test_initial_program_artifacts_are_stored(self): instance = mock_ppc.return_value instance.start.return_value = None instance.stop.return_value = None + instance.produced_no_candidates = False asyncio.run(controller.run(iterations=1)) @@ -118,6 +119,7 @@ def test_no_artifacts_when_evaluator_returns_none(self): ): mock_ppc.return_value.start.return_value = None mock_ppc.return_value.stop.return_value = None + mock_ppc.return_value.produced_no_candidates = False asyncio.run(controller.run(iterations=1)) self.assertEqual(len(controller.database.programs), 1) diff --git a/tests/test_island_system_messages.py b/tests/test_island_system_messages.py index 998d6c6b5..c6cc7cb64 100644 --- a/tests/test_island_system_messages.py +++ b/tests/test_island_system_messages.py @@ -136,6 +136,7 @@ def test_checkpoint_resume_with_same_config_keeps_override(self): ) with patch("openevolve.controller.ProcessParallelController") as parallel: parallel.return_value.run_evolution = AsyncMock(return_value=None) + parallel.return_value.produced_no_candidates = False asyncio.run(resumed.run(iterations=1, checkpoint_path=str(checkpoint))) self.assertEqual(resumed.database.last_iteration, 3) diff --git a/tests/test_utils.py b/tests/test_utils.py index 5482b24e3..c5293588f 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -143,6 +143,14 @@ def get_integration_config(port: int = DEFAULT_PORT) -> Config: # room for a small code diff. config.llm.max_tokens = 256 + # Use full rewrites rather than diffs. dhara-250m never emits a well-formed + # SEARCH/REPLACE block, so in diff mode every iteration ends with "No valid + # diffs found" and no candidate is produced. Since #288 a run in which no + # iteration produces a candidate raises EvolutionFailedError. Full-rewrite + # parsing falls back to the raw response, so each iteration yields a candidate + # and these tests exercise the full evaluate/add/migrate/checkpoint path. + config.diff_based_evolution = False + # Configure to use optillm server base_url = f"http://localhost:{port}/v1" config.llm.api_base = base_url