From 9875aaa2d1c05ea04f234b1a668b673e670e199a Mon Sep 17 00:00:00 2001 From: SUPR1M3 Date: Sat, 24 Oct 2026 04:36:35 -0400 Subject: [PATCH 1/3] Fix: Bug 288 + Tests --- openevolve/__init__.py | 3 +- openevolve/cli.py | 7 +- openevolve/controller.py | 17 ++ openevolve/process_parallel.py | 32 +++ tests/test_all_iterations_failed.py | 272 ++++++++++++++++++++++++ tests/test_checkpoint_resume.py | 5 + tests/test_initial_program_artifacts.py | 2 + tests/test_island_system_messages.py | 1 + 8 files changed, 337 insertions(+), 2 deletions(-) create mode 100644 tests/test_all_iterations_failed.py diff --git a/openevolve/__init__.py b/openevolve/__init__.py index 32972f5f09..947b2fa4f0 100644 --- a/openevolve/__init__.py +++ b/openevolve/__init__.py @@ -12,10 +12,11 @@ ) from openevolve.coevolution import CoevolutionResult, Opponent, Population, run_coevolution from openevolve.config import Config -from openevolve.controller import OpenEvolve +from openevolve.controller import EvolutionFailedError, OpenEvolve __all__ = [ "Config", + "EvolutionFailedError", "OpenEvolve", "__version__", "run_evolution", diff --git a/openevolve/cli.py b/openevolve/cli.py index a0e5bd1e11..26d5275be4 100644 --- a/openevolve/cli.py +++ b/openevolve/cli.py @@ -10,7 +10,7 @@ import sys from typing import Any, Dict, List, Optional -from openevolve import OpenEvolve +from openevolve import EvolutionFailedError, OpenEvolve from openevolve.config import Config, load_config logger = logging.getLogger(__name__) @@ -224,6 +224,11 @@ async def main_async() -> int: return 0 + except EvolutionFailedError as e: + # Expected failure mode with an actionable message, no traceback needed + print(f"\n{e}") + return 1 + except Exception as e: print(f"Error: {str(e)}") import traceback diff --git a/openevolve/controller.py b/openevolve/controller.py index 6b6a00684b..c0e5d386dd 100644 --- a/openevolve/controller.py +++ b/openevolve/controller.py @@ -28,6 +28,11 @@ logger = logging.getLogger(__name__) +class EvolutionFailedError(RuntimeError): + """Raised when every iteration of a run failed without producing a candidate + program, e.g. because the LLM endpoint is unreachable (issue #288)""" + + class OpenEvolve: """ Main controller for OpenEvolve @@ -261,6 +266,9 @@ async def run( Returns: Best program found + + Raises: + EvolutionFailedError: If iterations ran but none produced a candidate program """ max_iterations = self.config.max_iterations if iterations is None else iterations # Determine starting iteration @@ -375,6 +383,15 @@ def force_exit_handler(signum, frame): evolution_start, evolution_iterations, target_score ) + # A run where nothing was generated must not end looking like a run that + # simply found nothing better than the seed (issue #288) + if self.parallel_controller.produced_no_candidates: + raise EvolutionFailedError( + f"Evolution failed: 0/{self.parallel_controller.completed_iterations} " + f"iterations produced a candidate program. " + f"Last error: {self.parallel_controller.last_iteration_error}" + ) + finally: # Clean up parallel processing resources if self.parallel_controller: diff --git a/openevolve/process_parallel.py b/openevolve/process_parallel.py index 8c2b29adfd..33c19303ee 100644 --- a/openevolve/process_parallel.py +++ b/openevolve/process_parallel.py @@ -487,6 +487,14 @@ def __init__( self.shutdown_event = mp.Event() self.early_stopping_triggered = False + # Per-run outcome counters, reset by run_evolution (issue #288) + self.completed_iterations = 0 + self.successful_iterations = 0 + self.last_iteration_error: Optional[str] = None + # True when the last run completed iterations but every one of them failed + # before producing a candidate program + self.produced_no_candidates = False + # Number of worker processes self.num_workers = config.evaluator.parallel_evaluations self.num_islands = config.database.num_islands @@ -691,6 +699,10 @@ async def run_evolution( next_iteration = current_iteration completed_iterations = 0 + # Iterations whose worker produced a candidate program. A run where this + # stays at zero (e.g. the LLM endpoint is down) made no progress at all. + successful_iterations = 0 + self.last_iteration_error = None total_prompt_tokens = 0 total_completion_tokens = 0 total_tokens = 0 @@ -752,10 +764,13 @@ async def run_evolution( token_str = f" | tokens: {tt} (prompt: {pt}, completion: {ct})" if result.error: + self.last_iteration_error = result.error logger.warning( f"Iteration {completed_iteration} error: {result.error}{token_str}" ) elif result.child_program_dict: + successful_iterations += 1 + # Reconstruct program from dict child_program = Program(**result.child_program_dict) @@ -971,6 +986,7 @@ async def run_evolution( except FutureTimeoutError as e: if getattr(e, "_openevolve_strategy_error", False): raise + self.last_iteration_error = f"Iteration timed out after {timeout_seconds}s" logger.error( f"⏰ Iteration {completed_iteration} timed out after {timeout_seconds}s " f"(evaluator timeout: {self.config.evaluator.timeout}s + 30s buffer). " @@ -981,6 +997,7 @@ async def run_evolution( except Exception as e: if getattr(e, "_openevolve_strategy_error", False): raise + self.last_iteration_error = str(e) logger.error(f"Error processing result from iteration {completed_iteration}: {e}") completed_iterations += 1 @@ -1029,6 +1046,21 @@ async def run_evolution( else: logger.info("✅ Evolution completed - Maximum iterations reached") + self.completed_iterations = completed_iterations + self.successful_iterations = successful_iterations + # An interrupted run is not a failed one. Evaluated here, before stop() + # sets the shutdown event as part of normal cleanup. + self.produced_no_candidates = ( + completed_iterations > 0 + and successful_iterations == 0 + and not self.shutdown_event.is_set() + ) + if self.produced_no_candidates: + logger.error( + f"❌ None of the {completed_iterations} iterations produced a candidate program. " + f"Last error: {self.last_iteration_error}" + ) + if total_llm_calls > 0: logger.info( f"📊 Total LLM Token Usage ({total_llm_calls} calls): " diff --git a/tests/test_all_iterations_failed.py b/tests/test_all_iterations_failed.py new file mode 100644 index 0000000000..136f88c527 --- /dev/null +++ b/tests/test_all_iterations_failed.py @@ -0,0 +1,272 @@ +""" +Tests for runs where every iteration fails (issue #288) + +When the LLM endpoint is down, every iteration errors out before producing a +candidate program. Such a run used to finish with "Evolution complete!", exit +code 0 and the untouched seed saved as the best program, which is +indistinguishable from a run that simply found nothing better. +""" + +import asyncio +import http.server +import json +import os +import shutil +import subprocess +import sys +import tempfile +import threading +import unittest +from unittest.mock import MagicMock, patch + +# Set dummy API key for testing +os.environ["OPENAI_API_KEY"] = "test" + +from openevolve import EvolutionFailedError, OpenEvolve +from openevolve.config import Config +from openevolve.database import Program, ProgramDatabase +from openevolve.process_parallel import ProcessParallelController, SerializableResult + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +EVALUATOR_CODE = """ +def evaluate(program_path): + return {"combined_score": 0.5} +""" + + +def _error_future(iteration, error="LLM generation failed: Connection error."): + future = MagicMock() + future.done.return_value = True + future.result.return_value = SerializableResult(error=error, iteration=iteration) + return future + + +def _child_future(iteration, island_id): + future = MagicMock() + future.done.return_value = True + future.result.return_value = SerializableResult( + child_program_dict={ + "id": f"child_{iteration}", + "code": f"def evolved_{iteration}(): return {iteration}", + "language": "python", + "parent_id": "seed", + "generation": 1, + "metrics": {"combined_score": 0.4}, + "iteration_found": iteration, + "metadata": {"changes": "test", "island": island_id}, + }, + parent_id="seed", + iteration_time=0.1, + iteration=iteration, + ) + return future + + +class TestControllerCountsCandidates(unittest.TestCase): + """ProcessParallelController must report whether any iteration produced a candidate""" + + def setUp(self): + self.test_dir = tempfile.mkdtemp() + self.eval_file = os.path.join(self.test_dir, "evaluator.py") + with open(self.eval_file, "w") as f: + f.write(EVALUATOR_CODE) + + self.config = Config() + self.config.evaluator.parallel_evaluations = 2 + self.config.evaluator.timeout = 10 + self.config.database.num_islands = 1 + self.config.database.in_memory = True + + self.database = ProgramDatabase(self.config.database) + self.database.add( + Program( + id="seed", + code="def seed(): return 0", + language="python", + metrics={"combined_score": 0.5}, + iteration_found=0, + ) + ) + + def tearDown(self): + shutil.rmtree(self.test_dir, ignore_errors=True) + + def _run(self, make_future, max_iterations=4, before_run=None): + async def run_test(): + controller = ProcessParallelController(self.config, self.eval_file, self.database) + with patch.object(controller, "_submit_iteration", side_effect=make_future): + controller.start() + try: + if before_run: + before_run(controller) + await controller.run_evolution( + start_iteration=1, max_iterations=max_iterations, target_score=None + ) + finally: + controller.stop() + return controller + + return asyncio.run(run_test()) + + def test_all_errors_produce_no_candidates(self): + controller = self._run(lambda i, island: _error_future(i)) + + self.assertTrue(controller.produced_no_candidates) + self.assertEqual(controller.completed_iterations, 4) + self.assertEqual(controller.successful_iterations, 0) + self.assertEqual( + controller.last_iteration_error, "LLM generation failed: Connection error." + ) + + def test_single_success_counts_as_progress(self): + """Intermittent failures are still tolerated as long as something was generated""" + controller = self._run( + lambda i, island: _child_future(i, island) if i == 3 else _error_future(i) + ) + + self.assertFalse(controller.produced_no_candidates) + self.assertEqual(controller.successful_iterations, 1) + + def test_shutdown_is_not_a_failure(self): + """An interrupted run is not reported as a failed one""" + controller = self._run( + lambda i, island: _error_future(i), + before_run=lambda c: c.request_shutdown(), + ) + + self.assertFalse(controller.produced_no_candidates) + + +class TestOpenEvolveRunFailsWithoutCandidates(unittest.TestCase): + """OpenEvolve.run must raise instead of presenting the seed as the evolved best""" + + def setUp(self): + self.test_dir = tempfile.mkdtemp() + self.program_file = os.path.join(self.test_dir, "initial_program.py") + with open(self.program_file, "w") as f: + f.write("# EVOLVE-BLOCK-START\ndef f():\n return 0\n# EVOLVE-BLOCK-END\n") + self.eval_file = os.path.join(self.test_dir, "evaluator.py") + with open(self.eval_file, "w") as f: + f.write(EVALUATOR_CODE) + self.output_dir = os.path.join(self.test_dir, "out") + + self.config = Config() + self.config.evaluator.parallel_evaluations = 1 + self.config.database.num_islands = 1 + self.config.database.in_memory = True + + def tearDown(self): + shutil.rmtree(self.test_dir, ignore_errors=True) + + def _run(self, make_future): + openevolve = OpenEvolve( + initial_program_path=self.program_file, + evaluation_file=self.eval_file, + config=self.config, + output_dir=self.output_dir, + ) + with patch.object(ProcessParallelController, "_submit_iteration", autospec=True) as m: + m.side_effect = lambda _self, i, island=None: make_future(i, island) + return asyncio.run(openevolve.run(iterations=3)) + + def test_raises_and_does_not_save_seed_as_best(self): + with self.assertRaises(EvolutionFailedError) as ctx: + self._run(lambda i, island: _error_future(i)) + + self.assertIn("0/3 iterations produced a candidate program", str(ctx.exception)) + self.assertIn("Connection error", str(ctx.exception)) + self.assertFalse(os.path.exists(os.path.join(self.output_dir, "best"))) + + def test_partial_success_still_returns_best(self): + best = self._run(lambda i, island: _child_future(i, island) if i == 2 else _error_future(i)) + + self.assertIsNotNone(best) + self.assertTrue(os.path.exists(os.path.join(self.output_dir, "best"))) + + +class _DeadEndpointHandler(http.server.BaseHTTPRequestHandler): + """Answers every LLM request with HTTP 404, like a missing deployment""" + + def do_POST(self): + self.rfile.read(int(self.headers.get("Content-Length", 0))) + body = json.dumps( + {"error": {"code": "DeploymentNotFound", "message": "deployment does not exist"}} + ).encode() + self.send_response(404) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args): + pass + + +class TestCLIExitCodeWithDeadEndpoint(unittest.TestCase): + """End-to-end CLI check against a dead LLM endpoint (reproduction from issue #288)""" + + def setUp(self): + self.server = http.server.HTTPServer(("127.0.0.1", 0), _DeadEndpointHandler) + threading.Thread(target=self.server.serve_forever, daemon=True).start() + self.test_dir = tempfile.mkdtemp() + + def tearDown(self): + self.server.shutdown() + self.server.server_close() + shutil.rmtree(self.test_dir, ignore_errors=True) + + def test_exits_non_zero_without_success_banner(self): + program_file = os.path.join(self.test_dir, "initial_program.py") + with open(program_file, "w") as f: + f.write("# EVOLVE-BLOCK-START\ndef f():\n return 0\n# EVOLVE-BLOCK-END\n") + eval_file = os.path.join(self.test_dir, "evaluator.py") + with open(eval_file, "w") as f: + f.write(EVALUATOR_CODE) + config_file = os.path.join(self.test_dir, "config.yaml") + with open(config_file, "w") as f: + f.write( + "llm:\n" + f" api_base: http://127.0.0.1:{self.server.server_port}/v1\n" + " primary_model: dead-model\n" + " retries: 0\n" + " retry_delay: 0\n" + " timeout: 5\n" + "database:\n" + " num_islands: 1\n" + "evaluator:\n" + " parallel_evaluations: 1\n" + ) + output_dir = os.path.join(self.test_dir, "out") + + env = dict(os.environ) + env.setdefault("OPENAI_API_KEY", "test-key-for-unit-tests") + result = subprocess.run( + [ + sys.executable, + "-m", + "openevolve.cli", + program_file, + eval_file, + "--config", + config_file, + "--iterations", + "2", + "--output", + output_dir, + ], + cwd=REPO_ROOT, + env=env, + capture_output=True, + text=True, + timeout=120, + ) + + self.assertEqual(result.returncode, 1, result.stdout + result.stderr) + self.assertIn("0/2 iterations produced a candidate program", result.stdout) + self.assertNotIn("Evolution complete!", result.stdout) + self.assertFalse(os.path.exists(os.path.join(output_dir, "best"))) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_checkpoint_resume.py b/tests/test_checkpoint_resume.py index 791d5d5d84..5d2e5662eb 100644 --- a/tests/test_checkpoint_resume.py +++ b/tests/test_checkpoint_resume.py @@ -114,6 +114,7 @@ async def run_test(): mock_controller.stop = Mock(return_value=None) mock_controller.shutdown_event = Mock() mock_controller.shutdown_event.is_set.return_value = False + mock_controller.produced_no_candidates = False mock_controller_class.return_value = mock_controller # Run for 0 iterations (just initialization) @@ -170,6 +171,7 @@ async def run_test(): mock_controller.stop = Mock(return_value=None) mock_controller.shutdown_event = Mock() mock_controller.shutdown_event.is_set.return_value = False + mock_controller.produced_no_candidates = False mock_controller_class.return_value = mock_controller # Run for 0 iterations (just initialization) @@ -225,6 +227,7 @@ async def run_test(): mock_controller.stop = Mock(return_value=None) mock_controller.shutdown_event = Mock() mock_controller.shutdown_event.is_set.return_value = False + mock_controller.produced_no_candidates = False mock_controller_class.return_value = mock_controller # Run for 0 iterations (just initialization) @@ -283,6 +286,7 @@ async def run_test(): mock_controller.stop = Mock(return_value=None) mock_controller.shutdown_event = Mock() mock_controller.shutdown_event.is_set.return_value = False + mock_controller.produced_no_candidates = False mock_controller_class.return_value = mock_controller # Run for 0 iterations (just initialization) @@ -324,6 +328,7 @@ async def run_test(): mock_parallel.start = MagicMock() mock_parallel.stop = MagicMock() mock_parallel.shutdown_event.is_set.return_value = False + mock_parallel.produced_no_candidates = False mock_parallel_class.return_value = mock_parallel # Run first time diff --git a/tests/test_initial_program_artifacts.py b/tests/test_initial_program_artifacts.py index 079ff9eea1..dcf4e47d7d 100644 --- a/tests/test_initial_program_artifacts.py +++ b/tests/test_initial_program_artifacts.py @@ -84,6 +84,7 @@ def test_initial_program_artifacts_are_stored(self): instance = mock_ppc.return_value instance.start.return_value = None instance.stop.return_value = None + instance.produced_no_candidates = False asyncio.run(controller.run(iterations=1)) @@ -118,6 +119,7 @@ def test_no_artifacts_when_evaluator_returns_none(self): ): mock_ppc.return_value.start.return_value = None mock_ppc.return_value.stop.return_value = None + mock_ppc.return_value.produced_no_candidates = False asyncio.run(controller.run(iterations=1)) self.assertEqual(len(controller.database.programs), 1) diff --git a/tests/test_island_system_messages.py b/tests/test_island_system_messages.py index 998d6c6b55..c6cc7cb64c 100644 --- a/tests/test_island_system_messages.py +++ b/tests/test_island_system_messages.py @@ -136,6 +136,7 @@ def test_checkpoint_resume_with_same_config_keeps_override(self): ) with patch("openevolve.controller.ProcessParallelController") as parallel: parallel.return_value.run_evolution = AsyncMock(return_value=None) + parallel.return_value.produced_no_candidates = False asyncio.run(resumed.run(iterations=1, checkpoint_path=str(checkpoint))) self.assertEqual(resumed.database.last_iteration, 3) From 735450dd4cd69de085b8046f2947464754d802e6 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 25 Oct 2026 11:06:36 +0800 Subject: [PATCH 2/3] Use full rewrites in integration tests so the small model yields candidates dhara-250m never emits a well-formed SEARCH/REPLACE block, so in diff mode every integration iteration ended with "No valid diffs found". The tests passed only because such runs finished like successful ones; with EvolutionFailedError (#288) all 17 real-LLM integration tests failed. Switch the shared integration config and the library API test config to full-rewrite mode, whose parser falls back to the raw response, so every iteration produces a candidate and the tests exercise the evaluate, add, migrate and checkpoint paths that diff mode never reached. That exposed a stale assertion in test_checkpoint_resume_functionality: checkpoints contain metadata.json and programs/, not database.json. The test now checks that layout and actually resumes from the checkpoint, verifying the saved programs load and the run continues past it. Co-Authored-By: Claude Opus 5.5 --- tests/integration/README.md | 3 +++ tests/integration/test_checkpoint_with_llm.py | 20 +++++++++++++++++-- tests/integration/test_library_api.py | 3 +++ tests/test_utils.py | 8 ++++++++ 4 files changed, 32 insertions(+), 2 deletions(-) diff --git a/tests/integration/README.md b/tests/integration/README.md index a36306488b..c933d808a5 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -50,6 +50,9 @@ All integration tests use: - **120 second timeout** per LLM call - **In-memory database** for speed - **Small iteration counts** (1-8 iterations) for CI compatibility +- **Full rewrites** (`diff_based_evolution: false`): dhara-250m does not produce valid + SEARCH/REPLACE diffs, and a run where no iteration produces a candidate raises + `EvolutionFailedError` (#288), so the tests use full-rewrite mode to always get candidates ## CI Behavior diff --git a/tests/integration/test_checkpoint_with_llm.py b/tests/integration/test_checkpoint_with_llm.py index e801f9cfbe..492fab3fc8 100644 --- a/tests/integration/test_checkpoint_with_llm.py +++ b/tests/integration/test_checkpoint_with_llm.py @@ -88,9 +88,25 @@ async def test_checkpoint_resume_functionality( latest_checkpoint = max(checkpoint_dirs, key=lambda d: int(d.name.split("_")[1])) checkpoint_iter = int(latest_checkpoint.name.split("_")[1]) - # Test resume (simplified - just verify the checkpoint directory structure) - assert (latest_checkpoint / "database.json").exists(), "Database checkpoint should exist" + # ProgramDatabase.save writes metadata.json plus one JSON file per program + assert (latest_checkpoint / "metadata.json").exists(), "Database metadata should exist" + saved_ids = {p.stem for p in (latest_checkpoint / "programs").glob("*.json")} + assert saved_ids, "Checkpoint should contain saved programs" print(f"Successfully created checkpoint at iteration {checkpoint_iter}") + + # Resume from the checkpoint and run a few more iterations + controller2 = OpenEvolve( + initial_program_path=str(test_program_file), + evaluation_file=str(test_evaluator_file), + config=evolution_config, + output_dir=str(evolution_output_dir) + ) + await controller2.run(iterations=3, checkpoint_path=str(latest_checkpoint)) + + assert saved_ids <= set(controller2.database.programs), \ + "Resumed database should contain every program from the checkpoint" + assert controller2.database.last_iteration > checkpoint_iter, \ + "Resumed run should continue past the checkpoint iteration" else: print("No checkpoints created (likely due to all iterations failing)") else: diff --git a/tests/integration/test_library_api.py b/tests/integration/test_library_api.py index d050d07bb6..e13aee52b4 100644 --- a/tests/integration/test_library_api.py +++ b/tests/integration/test_library_api.py @@ -36,6 +36,9 @@ def _get_library_test_config(port: int = 8000) -> Config: # Bound generation length so the (non-stopping) dhara model does not ramble to # the 4096-token default on every call (see tests/test_utils.py for details). config.llm.max_tokens = 256 + # Full rewrites so the small model always yields a candidate (see + # get_integration_config in tests/test_utils.py); runs with no candidates fail. + config.diff_based_evolution = False config.llm.models = [ LLMModelConfig( name=TEST_MODEL, diff --git a/tests/test_utils.py b/tests/test_utils.py index 5482b24e3e..c5293588f0 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -143,6 +143,14 @@ def get_integration_config(port: int = DEFAULT_PORT) -> Config: # room for a small code diff. config.llm.max_tokens = 256 + # Use full rewrites rather than diffs. dhara-250m never emits a well-formed + # SEARCH/REPLACE block, so in diff mode every iteration ends with "No valid + # diffs found" and no candidate is produced. Since #288 a run in which no + # iteration produces a candidate raises EvolutionFailedError. Full-rewrite + # parsing falls back to the raw response, so each iteration yields a candidate + # and these tests exercise the full evaluate/add/migrate/checkpoint path. + config.diff_based_evolution = False + # Configure to use optillm server base_url = f"http://localhost:{port}/v1" config.llm.api_base = base_url From 0ae9eb9313e5feaaedc17d7f44948b1354d9370a Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 25 Oct 2026 11:44:25 +0800 Subject: [PATCH 3/3] Run library API integration tests for three iterations test_evolve_code_real_integration ran a single iteration, and dhara-250m occasionally returns an empty reply ("No valid code found in response"), which leaves the run with no candidate and now raises EvolutionFailedError. Run each library API evolution for three iterations so one empty reply does not fail the test. Co-Authored-By: Claude Opus 5.5 --- tests/integration/test_library_api.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/integration/test_library_api.py b/tests/integration/test_library_api.py index e13aee52b4..5633536a23 100644 --- a/tests/integration/test_library_api.py +++ b/tests/integration/test_library_api.py @@ -77,7 +77,7 @@ def simple_multiply(x, y): result = evolve_function( simple_multiply, test_cases, - iterations=2, # Very small number for CI speed + iterations=3, # A few iterations: one empty reply from the small model would leave no candidate output_dir=str(temp_workspace / "evolve_function_output"), cleanup=False, # Keep files for inspection config=_get_library_test_config(optillm_server["port"]), @@ -165,7 +165,7 @@ def fibonacci_evaluator(program_path): result = evolve_code( initial_code, fibonacci_evaluator, - iterations=1, # Minimal for CI speed + iterations=3, # A few iterations: one empty reply from the small model would leave no candidate output_dir=str(temp_workspace / "evolve_code_output"), cleanup=False, # Keep output directory config=_get_library_test_config(optillm_server["port"]), @@ -260,7 +260,7 @@ def evaluate(program_path): result = run_evolution( initial_program=str(initial_program), evaluator=str(evaluator_file), - iterations=1, # Minimal for CI speed + iterations=3, # A few iterations: one empty reply from the small model would leave no candidate output_dir=str(temp_workspace / "run_evolution_output"), cleanup=False, # Keep output directory config=_get_library_test_config(optillm_server["port"]), @@ -287,7 +287,7 @@ def evaluate(program_path): result2 = run_evolution( initial_program=initial_program.read_text(), evaluator=lambda path: {"score": 0.8, "test": "passed"}, # Simple callable evaluator - iterations=1, + iterations=3, # A few iterations: one empty reply from the small model would leave no candidate output_dir=str(temp_workspace / "run_evolution_string_output"), cleanup=False, # Keep output directory config=_get_library_test_config(optillm_server["port"]),