From 503f5c8ed569a4fa2d1284d91abfdace82fba0c8 Mon Sep 17 00:00:00 2001 From: PetrichorHlacyon <2262345951@qq.com> Date: Wed, 22 Apr 2026 12:33:15 +0800 Subject: [PATCH 1/3] run_evolution function with checkpoint parameter --- openevolve/api.py | 8 ++++++-- openevolve/controller.py | 1 - 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/openevolve/api.py b/openevolve/api.py index 65c3702064..a265b19ab5 100644 --- a/openevolve/api.py +++ b/openevolve/api.py @@ -37,6 +37,8 @@ def run_evolution( iterations: Optional[int] = None, output_dir: Optional[str] = None, cleanup: bool = True, + target_score: Optional[float] = None, + checkpoint_path: Optional[str] = None, ) -> EvolutionResult: """ Run evolution with flexible inputs - the main library API @@ -90,7 +92,7 @@ def my_evaluator(program_path): ) """ return asyncio.run( - _run_evolution_async(initial_program, evaluator, config, iterations, output_dir, cleanup) + _run_evolution_async(initial_program, evaluator, config, iterations, output_dir, cleanup, target_score, checkpoint_path) ) @@ -101,6 +103,8 @@ async def _run_evolution_async( iterations: Optional[int], output_dir: Optional[str], cleanup: bool, + target_score: Optional[float] = None, + checkpoint_path: Optional[str] = None, ) -> EvolutionResult: """Async implementation of run_evolution""" @@ -156,7 +160,7 @@ async def _run_evolution_async( output_dir=actual_output_dir, ) - best_program = await controller.run(iterations=iterations) + best_program = await controller.run(iterations=iterations,target_score=target_score,checkpoint_path=checkpoint_path) # Prepare result best_score = 0.0 diff --git a/openevolve/controller.py b/openevolve/controller.py index 01ffec73c3..7d58b502c5 100644 --- a/openevolve/controller.py +++ b/openevolve/controller.py @@ -264,7 +264,6 @@ async def run( Best program found """ max_iterations = iterations or self.config.max_iterations - # Determine starting iteration start_iteration = 0 if checkpoint_path and os.path.exists(checkpoint_path): From 7151d00b4f0342dbf2fbc15b4e525cb67094a3ad Mon Sep 17 00:00:00 2001 From: Asankhaya Sharma Date: Tue, 14 Jul 2026 07:56:32 +0800 Subject: [PATCH 2/3] Add background_blur example: evolving a hot function behind a hard quality gate A self-contained example of the "optimise a real hot function" pattern, where the score is a speedup but fidelity is a hard gate: a fast-but-wrong candidate scores zero. Deterministic and dependency-light (numpy only) - the video sequence is synthesised from a fixed seed, so there is no webcam, GPU, model or dataset. The seed is a correct-but-slow O(k^2) 2D Gaussian convolution. Over 100 iterations the search reached 62x over that seed (2.6x over a hand-written separable+fp32 implementation), stacking separable convolution, a half-resolution blur pipeline, float32, batching across the sequence, and a lerp composite. 28 of 100 candidates were rejected outright by the quality gate. Two things this example exists to demonstrate: - The obvious quality gate is exploitable. Grading mean SSIM and worst-FRAME SSIM lets a "blur frame 0's background and reuse it forever" candidate score 47x while leaving a visible person-shaped ghost: it damages one region, and the frame average hides it. Grading the worst 16x16 REGION catches it (0.74 vs >0.97 for legitimate approximations). test_gaming.py encodes the cheats as tests so they must lose. - Timing-as-fitness needs care. Measuring the baseline once and caching it lets a transiently loaded machine inflate every speedup that follows (this reported 82x for a program that is really 62x). The evaluator now interleaves baseline and candidate in the same call, warms up, and takes best-of-N. Uses the cascade (cheap smoke -> quality gate -> benchmark) so timing is only spent on candidates that are already correct, artifacts to tell the model WHY it was rejected, and a (complexity, ssim) MAP-Elites grid to keep exact-and-fast and approximate-and-faster strategies both alive. Co-Authored-By: Claude Opus 4.8 (1M context) --- examples/background_blur/README.md | 208 ++++++++++++ examples/background_blur/config.yaml | 90 +++++ examples/background_blur/evaluator.py | 345 ++++++++++++++++++++ examples/background_blur/fixtures.py | 235 +++++++++++++ examples/background_blur/initial_program.py | 73 +++++ examples/background_blur/test_gaming.py | 167 ++++++++++ 6 files changed, 1118 insertions(+) create mode 100644 examples/background_blur/README.md create mode 100644 examples/background_blur/config.yaml create mode 100644 examples/background_blur/evaluator.py create mode 100644 examples/background_blur/fixtures.py create mode 100644 examples/background_blur/initial_program.py create mode 100644 examples/background_blur/test_gaming.py diff --git a/examples/background_blur/README.md b/examples/background_blur/README.md new file mode 100644 index 0000000000..571926a1e1 --- /dev/null +++ b/examples/background_blur/README.md @@ -0,0 +1,208 @@ +# Portrait background blur — evolving a hot function behind a hard quality gate + +Optimise the per-frame background blur of a video pipeline. The score is the +**speedup**, but fidelity is a **hard gate**: a candidate that is fast and wrong +scores exactly zero. + +This is the "optimise a real hot function" pattern. The interesting part is not the +LLM — it is the **evaluator**. Getting the gate right *is* the job. + +``` +speedup = baseline_time / candidate_time + +reject if mean SSIM < 0.98 (overall fidelity) + or worst-frame SSIM < 0.95 (no bad frames) + or worst-REGION SSIM < 0.90 (no bad regions) +otherwise score = speedup +``` + +Everything is deterministic and self-contained: the "video" is synthesised from a +fixed seed, so there is no webcam, no GPU, no ML model and no dataset to download. +`numpy` is the only dependency. + +## The task + +For each frame, blur the background and composite the sharp person back on top, +using a supplied segmentation mask: + +```python +blurred = gaussian_blur(frame, sigma) +out = mask * frame + (1 - mask) * blurred +``` + +The seed (`initial_program.py`) is **correct but slow**: it convolves with the full +2D Gaussian directly — O(k²) passes per frame. It is the honest implementation you +would write first, and it is the thing to beat. + +## Run it + +```bash +export OPENROUTER_API_KEY=sk-or-... + +python openevolve-run.py \ + examples/background_blur/initial_program.py \ + examples/background_blur/evaluator.py \ + --config examples/background_blur/config.yaml \ + --iterations 100 +``` + +`config.yaml` uses OpenRouter with a cheap model (`google/gemini-2.5-flash-lite`). +Any OpenAI-compatible endpoint works — point `api_base` at a local optillm server to +run with no hosted API and no key at all. + +## Results + +100 iterations. Timings are best-of-N on an idle machine (see the timing note below — +this matters more than you would think). + +| | ms/frame | speedup | mean SSIM | worst-region | +|---|---:|---:|---:|---:| +| seed — naive O(k²) convolution | 33.4 | 1.0x | 1.0000 | 1.0000 | +| expert — hand-written separable + fp32 | 1.46 | 22.8x | 1.0000 | 1.0000 | +| **evolved (OpenEvolve)** | **0.54** | **62x** | 0.9915 | 0.9723 | + +**Read those two speedup columns carefully.** 62x is against a baseline that was +*written to be bad on purpose*, so it flatters the result. The number that actually +means something is the comparison against a competent implementation: + +> **The evolved solution is 2.6x faster than a hand-written separable+fp32 blur** — +> and it gets there by spending part of the fidelity budget the gate allows +> (SSIM 0.9915 vs the expert's exact 1.0000). It is a real win, not a free one. + +28 of the 100 candidates were **rejected outright by the quality gate**. The gate is +not decoration. + +### What it discovered + +The winning program stacks five distinct optimisations, one of which (blurring at +reduced resolution) is the same class of trick reported for this problem elsewhere: + +```python +# 1. separable Gaussian: two 1D passes instead of k^2 +# 2. blur at HALF resolution, then upsample (quarter the pixels) +downsample = sigma >= 1.0 +sigma_eff = sigma / 2.0 # rescale sigma for the small domain +low = frames_arr[:, ::2, ::2, :] +blurred_low = _separable_blur_batch(low, kernel) +blurred = np.repeat(np.repeat(blurred_low, 2, axis=1), 2, axis=2) + +# 3. float32 rather than float64 (halves memory traffic) +# 4. BATCH the whole sequence into (N, H, W, C) and blur it in one vectorised pass +frames_arr = np.stack(frames).astype(np.float32) + +# 5. lerp composite: one multiply fewer than m*f + (1-m)*b +composited = blurred + masks_arr[..., None] * (frames_arr - blurred) +``` + +Nobody told it to batch across frames. It is search, not magic — but it is real search. + +## Attack your own evaluator first + +Before spending a single token, prove the cheats lose: + +```bash +python examples/background_blur/test_gaming.py +``` + +Each test is a candidate that is **fast but wrong**, and must score 0: + +| cheat | why it is tempting | +|---|---| +| return the input untouched | infinitely fast | +| blur everything, ignore the mask | skips compositing | +| under-blur with a tiny kernel | far cheaper than the real sigma | +| blur frame 0's background, reuse it forever | ~50x faster | + +...plus one honest optimisation (separable convolution) that **must** be accepted. + +### The gate we would have shipped was broken + +The first version of this evaluator used the obvious gate — mean SSIM ≥ 0.98 and +worst-frame SSIM ≥ 0.95. The **stale-background** cheat sailed straight through and +scored **47x**: + +| candidate | mean | worst-frame | **worst-region** | +|---|---:|---:|---:| +| separable (exact) | 1.0000 | 1.0000 | 1.0000 | +| **CHEAT: stale background** | 0.9871 | **0.9806** | **0.7445** | +| half-res blur (legitimate) | 0.9915 | 0.9912 | 0.9723 | +| 3× box blur (legitimate) | 0.9949 | 0.9944 | 0.9901 | + +Reusing one blurred background leaves a **person-shaped ghost** where the person used +to be. A human sees it instantly — but it damages a small patch, and whole-image SSIM +averages it away. Both frame-level gates pass. + +The fix is to grade the **worst 16×16 block**, not the whole frame. The ghost region +scores 0.74 while genuine approximations stay above 0.97, so `worst_region ≥ 0.90` +separates them cleanly. + +**The lesson generalises:** an aggregate metric hides localised damage. If your quality +bar is an average, the search will find the thing your average cannot see. Write the +cheats yourself and make sure they lose. + +## Timing as fitness is its own trap + +`parallel_evaluations: 1` is necessary — concurrent evaluations contend for CPU and +corrupt the measurement — but it is **not sufficient**. + +The first version of this evaluator measured the baseline **once**, cached it, and +compared every later candidate against it. That one measurement happened while the +machine was busy, so the baseline was inflated (95.6 ms/frame vs a true 33.4) — and +therefore *every speedup reported during the run was inflated too*. The winner was +reported at **82x**; it is really **62x**. + +The fix (`_benchmark()` in `evaluator.py`): + +- measure baseline and candidate **interleaved, in the same call**, so slow drift + cancels instead of accumulating into the ratio; +- **warm up** before timing; +- take the **minimum** of N runs — background load can only ever *add* time, so the + minimum is the best estimate of the true cost, for both sides. + +If your fitness is a wall-clock number, re-measure your final result on an idle +machine before you believe it. + +## Why OpenEvolve fits this problem + +**Cascade evaluation — don't benchmark garbage.** Timing is the expensive part, so it +is the last thing we do: + +``` +stage 1 cheap smoke test (2 frames: shape, finite, did you blur at all?) +stage 2 the quality gate (all frames: SSIM vs the reference) +stage 3 the benchmark (only for candidates that earned it) +``` + +**Artifacts — tell the model *why* it failed.** A scalar score says "0". OpenEvolve's +artifact side-channel hands the next prompt the actual reason: + +> QUALITY GATE FAILED: mean SSIM 0.9871 (needs >= 0.98), worst-frame SSIM 0.9806 on +> frame 6, worst-REGION SSIM 0.7445 (needs >= 0.90). ... do not reuse a stale +> background (it leaves a person-shaped ghost that wrecks one region while barely +> moving the frame average). + +**MAP-Elites — keep rival strategies alive.** The grid is `(complexity, ssim)`, so +"exact and fast" (separable, SSIM 1.0) and "approximate and faster" (half-res, SSIM +0.99) both survive instead of the population collapsing onto whichever appeared first. + +## Files + +| file | what it is | +|---|---| +| `initial_program.py` | the seed — correct, slow, `EVOLVE-BLOCK` markers | +| `evaluator.py` | 3-stage cascade, the hard gate, interleaved timing, artifacts | +| `fixtures.py` | deterministic scene, trusted reference, SSIM + worst-region SSIM | +| `test_gaming.py` | adversarial tests — the cheats must lose | +| `config.yaml` | OpenRouter endpoint, MAP-Elites grid, `parallel_evaluations: 1` | + +## Adapting this to your own hot function + +1. Wrap the function in `EVOLVE-BLOCK-START` / `EVOLVE-BLOCK-END`. +2. Write a **reference** implementation the evaluator owns (never handed to the + candidate) and a **fidelity metric** with a hard threshold. +3. Write the cheats and prove they score zero. Then go looking for the *localised* + cheat your aggregate metric cannot see. +4. Put the expensive measurement in the last cascade stage, and measure it fairly. +5. Return artifacts explaining every rejection. +6. Compare against a **competent** implementation, not just your slow seed — that is + the only number that means anything. diff --git a/examples/background_blur/config.yaml b/examples/background_blur/config.yaml new file mode 100644 index 0000000000..3e022ad338 --- /dev/null +++ b/examples/background_blur/config.yaml @@ -0,0 +1,90 @@ +# Portrait background blur - optimise a hot function behind a hard quality gate. +# +# Uses OpenRouter with a cheap, fast model. Set your key first: +# +# export OPENROUTER_API_KEY=sk-or-... +# +# Any OpenAI-compatible endpoint works. To run fully locally with no hosted API and +# no key, point `api_base` at a local optillm server instead, e.g.: +# +# api_base: "http://localhost:8000/v1" +# api_key: "dummy" +# models: [{ name: "optillm", api_base: "http://localhost:8000/v1", weight: 1.0 }] + +max_iterations: 100 +checkpoint_interval: 5 +diff_based_evolution: false # full rewrites: the win here is restructuring the blur +language: python +max_code_length: 40000 + +llm: + api_base: "https://openrouter.ai/api/v1" + api_key: "${OPENROUTER_API_KEY}" + max_tokens: 6000 + temperature: 0.9 + timeout: 600 + retries: 6 + retry_delay: 8 + models: + - name: "google/gemini-2.5-flash-lite" + api_base: "https://openrouter.ai/api/v1" + api_key: "${OPENROUTER_API_KEY}" + weight: 1.0 + +prompt: + system_message: | + You are an expert at high-performance numerical Python (NumPy). + + You are optimising the hot function of a real-time video pipeline: a portrait-mode + background blur. For every frame it must blur the background with a Gaussian of the + given sigma and composite the (sharp) person back on top using the supplied mask: + + blurred = gaussian_blur(frame, sigma) + out = mask * frame + (1 - mask) * blurred + + The current implementation is correct but slow: it convolves with the full 2D + Gaussian kernel directly, costing O(k^2) passes per frame. + + YOUR SCORE IS THE SPEEDUP. But speed only counts if the output still matches the + reference blur. Fidelity is a HARD GATE, not a trade-off: + + mean SSIM >= 0.98 + worst-frame SSIM >= 0.95 + worst-REGION SSIM >= 0.90 (worst 16x16 block - catches localised artefacts) + + Fail any of those and you score ZERO no matter how fast you are. So do NOT: + - skip the blur, or under-blur with a kernel smaller than sigma demands + - blur the person (always respect the mask) + - reuse a stale background across frames (the background genuinely changes; + a stale background leaves a person-shaped ghost that destroys one region + while barely moving the frame average) + + Legitimate ways to go faster include (but are not limited to): exploiting the + separability of the Gaussian (two 1D passes instead of k^2), blurring at reduced + resolution and upsampling, approximating the Gaussian with repeated box blurs via + summed-area tables/cumsum, FFT convolution, avoiding needless float64 work and + redundant copies, and vectorising across the sequence. + + Use only numpy and the Python standard library. Keep the public entry point + `process_sequence(frames, masks, sigma)` with the same signature and return type. + +database: + population_size: 30 + num_islands: 2 + # MAP-Elites grid: spread candidates across code complexity and how much fidelity + # they spend, so we keep BOTH the exact-and-fast and the approximate-and-faster + # strategies alive instead of collapsing onto one. + feature_dimensions: + - complexity + - ssim + +evaluator: + timeout: 600 + cascade_evaluation: true + # Stage 1 (smoke) and stage 2 (quality gate) each return combined_score 1.0 on pass + # and 0.0 on reject, so a 0.5 threshold means "only benchmark what is already correct". + cascade_thresholds: [0.5, 0.5] + # IMPORTANT: fitness here is a TIMING measurement. Concurrent evaluations contend for + # CPU and corrupt the benchmark, so this MUST stay at 1. Note that even serial + # evaluation is not sufficient on its own - see the timing notes in the README. + parallel_evaluations: 1 diff --git a/examples/background_blur/evaluator.py b/examples/background_blur/evaluator.py new file mode 100644 index 0000000000..a7192932ad --- /dev/null +++ b/examples/background_blur/evaluator.py @@ -0,0 +1,345 @@ +""" +Evaluator for the portrait background-blur task. + +The scoring rule, in one line: + + Quality is a GATE, not a trade-off. The only way to score is to look like the + reference and be faster. + + speedup = baseline_time / candidate_time + reject if mean_ssim < 0.98 (overall fidelity) + or worst_frame_ssim < 0.95 (no bad frames) + or worst_region_ssim < 0.90 (no bad REGIONS - see below) + otherwise score = speedup + +That gate is what stops the search from "winning" by simply doing less work. + +The worst-REGION term is the one people forget. Mean SSIM is blind to localised +damage: a candidate that blurs frame 0's background once and reuses it forever +leaves a person-shaped ghost, yet still scores mean 0.987 / worst-frame 0.981 and +sails through the first two gates. It scored 47x before the region gate existed. +Attack your own scorer (see test_gaming.py) before you spend a single LLM token. + +Structured as an OpenEvolve CASCADE so we never spend a timing benchmark on a +candidate that is already wrong: + + stage 1 cheap correctness smoke (shape/dtype/finite, and NOT a no-op) + stage 2 the hard quality gate (per-frame SSIM vs the reference) + stage 3 the timing benchmark (only for candidates that earned it) + +Every rejection returns ARTIFACTS explaining exactly what went wrong (which frame, +what SSIM, whether the output was simply the untouched input). The LLM sees the +reason, not just a number. +""" + +import importlib.util +import os +import sys +import time +import traceback + +import numpy as np + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from fixtures import ( # noqa: E402 + N_FRAMES, + SIGMA, + make_sequence, + naive_process, + reference_process, + ssim, + worst_region_ssim, +) +from openevolve.evaluation_result import EvaluationResult # noqa: E402 + +# --------------------------------------------------------------------------- +# The hard quality gate +# --------------------------------------------------------------------------- +MEAN_SSIM_GATE = 0.98 +WORST_SSIM_GATE = 0.95 + +# Whole-image SSIM is blind to LOCALISED damage. A candidate that blurs frame 0's +# background once and reuses it for the whole sequence leaves a person-shaped ghost +# where the person used to be - and still scores mean 0.987 / worst-frame 0.981, +# sailing through the two gates above. (It scored 47x before this gate existed.) +# Grading the worst 16x16 block catches it: the ghost region scores 0.74, while +# legitimate approximations (half-res blur, 3x box blur) stay above 0.97. +WORST_REGION_GATE = 0.90 + +# A candidate that just returns the input unchanged still scores fairly high SSIM +# (most of the frame is background that is only mildly blurred). This floor is a +# cheap stage-1 tripwire for "did you actually do anything at all". +NOOP_SSIM_CEILING = 0.995 + +TIMING_REPEATS = 3 + +# Computed once per evaluator process and reused (see README: the baseline must be +# measured under the same conditions as the candidate, serially). +_CACHE = {} + + +def _fixtures(): + if "frames" not in _CACHE: + frames, masks = make_sequence() + _CACHE["frames"] = frames + _CACHE["masks"] = masks + _CACHE["golden"] = reference_process(frames, masks, SIGMA) + # SSIM of the *untouched input* vs the reference: anything at or above this + # did not really blur anything. + _CACHE["identity_ssim"] = float( + np.mean([ssim(f, g) for f, g in zip(frames, _CACHE["golden"])]) + ) + return _CACHE["frames"], _CACHE["masks"], _CACHE["golden"] + + +def _benchmark(candidate_fn, frames, masks, sigma): + """Time the baseline and the candidate FAIRLY, and return (baseline, candidate, out). + + Two things matter here, and getting either wrong silently corrupts every score: + + 1. INTERLEAVE. Do not measure the baseline once, cache it, and compare every + future candidate against it. If that one measurement happened under transient + load it is inflated forever, and so is every speedup you report. (This is not + hypothetical: an earlier version of this evaluator did exactly that and + reported 82x for a program that is really 61x.) Measuring both back-to-back + in the same loop cancels slow drift. + + 2. WARM UP, then take the MINIMUM of N runs. Background load can only ever ADD + time, so the minimum is the best estimate of the true cost - for both sides. + """ + # Warm-up (allocator, pages, BLAS threads) - not measured. + naive_process(frames[:1], masks[:1], sigma) + candidate_fn(frames[:1], masks[:1], sigma) + + t_base = float("inf") + t_cand = float("inf") + out = None + for _ in range(TIMING_REPEATS): + t0 = time.perf_counter() + naive_process(frames, masks, sigma) + t_base = min(t_base, time.perf_counter() - t0) + + t0 = time.perf_counter() + out = candidate_fn(frames, masks, sigma) + t_cand = min(t_cand, time.perf_counter() - t0) + + return t_base, t_cand, out + + +def _load(program_path): + spec = importlib.util.spec_from_file_location("candidate", program_path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + if not hasattr(module, "process_sequence"): + raise AttributeError("program must define process_sequence(frames, masks, sigma)") + return module + + +def _check_shapes(out, frames): + """Structural validation. Returns an error string, or None if OK.""" + if out is None: + return "process_sequence returned None" + if len(out) != len(frames): + return f"expected {len(frames)} output frames, got {len(out)}" + for i, (o, f) in enumerate(zip(out, frames)): + o = np.asarray(o) + if o.shape != f.shape: + return f"frame {i}: expected shape {f.shape}, got {o.shape}" + if not np.all(np.isfinite(o)): + return f"frame {i}: output contains NaN or Inf" + return None + + +def _score_quality(out, golden): + """Returns (mean_ssim, worst_frame_ssim, worst_frame_idx, worst_region_ssim, per_frame).""" + outs = [np.asarray(o, dtype=np.float32) for o in out] + per_frame = [ssim(o, g) for o, g in zip(outs, golden)] + worst_region = min(worst_region_ssim(o, g) for o, g in zip(outs, golden)) + return ( + float(np.mean(per_frame)), + float(np.min(per_frame)), + int(np.argmin(per_frame)), + float(worst_region), + per_frame, + ) + + +def _fail(stage, reason, **extra): + """Rejection. Always carries `ssim` because it is a MAP-Elites feature dimension + (a program missing a declared feature dimension makes the database raise).""" + metrics = {"combined_score": 0.0, "ssim": 0.0} + metrics.update({k: float(v) for k, v in extra.items() if isinstance(v, (int, float))}) + artifacts = {"failure_stage": stage, "failure_reason": reason} + artifacts.update({k: str(v) for k, v in extra.items()}) + return EvaluationResult(metrics=metrics, artifacts=artifacts) + + +# --------------------------------------------------------------------------- +# Stage 1 - cheap correctness smoke test +# --------------------------------------------------------------------------- +def evaluate_stage1(program_path): + """Fast reject: does it run, produce the right shape, and actually blur?""" + try: + module = _load(program_path) + frames, masks, golden = _fixtures() + + # Only the first two frames - this stage must stay cheap. + f2, m2, g2 = frames[:2], masks[:2], golden[:2] + out = module.process_sequence(f2, m2, SIGMA) + + err = _check_shapes(out, f2) + if err: + return _fail("stage1", err) + + mean_ssim, _, _, _, _ = _score_quality(out, g2) + + # Anti-gaming tripwire: returning the input untouched is fast and scores + # deceptively well. Reject it here, before it ever reaches the benchmark. + identity_ssim = float(np.mean([ssim(f, g) for f, g in zip(f2, g2)])) + if mean_ssim >= NOOP_SSIM_CEILING and mean_ssim <= identity_ssim + 1e-6: + return _fail( + "stage1", + "output is (nearly) the untouched input - the background was not blurred", + ssim=mean_ssim, + identity_ssim=identity_ssim, + ) + + return EvaluationResult( + metrics={"combined_score": 1.0, "ssim": float(mean_ssim)}, + artifacts={"stage1": f"ok (ssim {mean_ssim:.4f} on 2 frames)"}, + ) + except Exception as e: + return _fail("stage1", f"exception: {e}", traceback=traceback.format_exc()) + + +# --------------------------------------------------------------------------- +# Stage 2 - the HARD QUALITY GATE +# --------------------------------------------------------------------------- +def evaluate_stage2(program_path): + """The gate. Fidelity is pass/fail; it is never traded against speed.""" + try: + module = _load(program_path) + frames, masks, golden = _fixtures() + + out = module.process_sequence(frames, masks, SIGMA) + err = _check_shapes(out, frames) + if err: + return _fail("stage2", err) + + mean_ssim, worst_ssim, worst_i, worst_region, per_frame = _score_quality(out, golden) + + if ( + mean_ssim < MEAN_SSIM_GATE + or worst_ssim < WORST_SSIM_GATE + or worst_region < WORST_REGION_GATE + ): + return _fail( + "stage2", + ( + f"QUALITY GATE FAILED: mean SSIM {mean_ssim:.4f} (needs >= {MEAN_SSIM_GATE}), " + f"worst-frame SSIM {worst_ssim:.4f} on frame {worst_i} " + f"(needs >= {WORST_SSIM_GATE}), worst-REGION SSIM {worst_region:.4f} " + f"(needs >= {WORST_REGION_GATE}). " + "The output no longer matches the reference blur. Speed is only counted " + "once the result is faithful on EVERY frame AND in EVERY region - do not " + "under-blur, skip frames, blur the person, or reuse a stale background " + "(a stale background leaves a person-shaped ghost that wrecks one region " + "while barely moving the frame average)." + ), + ssim=mean_ssim, + worst_ssim=worst_ssim, + worst_frame=worst_i, + worst_region_ssim=worst_region, + per_frame_ssim=", ".join(f"{s:.4f}" for s in per_frame), + ) + + return EvaluationResult( + metrics={ + "combined_score": 1.0, + "ssim": mean_ssim, + "worst_ssim": worst_ssim, + "worst_region_ssim": worst_region, + }, + artifacts={ + "stage2": ( + f"quality gate PASSED (mean {mean_ssim:.4f}, worst-frame {worst_ssim:.4f}, " + f"worst-region {worst_region:.4f})" + ) + }, + ) + except Exception as e: + return _fail("stage2", f"exception: {e}", traceback=traceback.format_exc()) + + +# --------------------------------------------------------------------------- +# Stage 3 - timing benchmark (earned only by candidates that passed the gate) +# --------------------------------------------------------------------------- +def evaluate_stage3(program_path): + """Measure the speedup. Fitness = speedup.""" + try: + module = _load(program_path) + frames, masks, golden = _fixtures() + + # Baseline and candidate are measured interleaved, in this same call, so a + # noisy machine cannot inflate the ratio. See _benchmark(). + baseline, best, out = _benchmark(module.process_sequence, frames, masks, SIGMA) + + # Re-verify quality on the timed output: speed must never be bought with + # fidelity, not even by a candidate that behaves differently under timing. + err = _check_shapes(out, frames) + if err: + return _fail("stage3", err) + mean_ssim, worst_ssim, worst_i, worst_region, _ = _score_quality(out, golden) + if ( + mean_ssim < MEAN_SSIM_GATE + or worst_ssim < WORST_SSIM_GATE + or worst_region < WORST_REGION_GATE + ): + return _fail( + "stage3", + f"quality regressed under timing: mean {mean_ssim:.4f}, " + f"worst-frame {worst_ssim:.4f}, worst-region {worst_region:.4f}", + ssim=mean_ssim, + worst_ssim=worst_ssim, + worst_frame=worst_i, + worst_region_ssim=worst_region, + ) + + speedup = baseline / best + ms_per_frame = 1000.0 * best / max(1, N_FRAMES) + baseline_ms = 1000.0 * baseline / max(1, N_FRAMES) + + return EvaluationResult( + metrics={ + # Fitness IS the speedup - but only reachable through the gate. + "combined_score": float(speedup), + "speedup": float(speedup), + "ssim": float(mean_ssim), + "worst_ssim": float(worst_ssim), + "worst_region_ssim": float(worst_region), + "ms_per_frame": float(ms_per_frame), + }, + artifacts={ + "result": ( + f"{speedup:.2f}x speedup | {ms_per_frame:.2f} ms/frame " + f"(baseline {baseline_ms:.2f} ms/frame) | mean SSIM {mean_ssim:.4f}, " + f"worst-frame {worst_ssim:.4f}, worst-region {worst_region:.4f}" + ) + }, + ) + except Exception as e: + return _fail("stage3", f"exception: {e}", traceback=traceback.format_exc()) + + +# --------------------------------------------------------------------------- +# Non-cascade entry point (used when cascade_evaluation: false) +# --------------------------------------------------------------------------- +def evaluate(program_path): + r1 = evaluate_stage1(program_path) + if r1.metrics.get("combined_score", 0.0) < 0.5: + return r1 + r2 = evaluate_stage2(program_path) + if r2.metrics.get("combined_score", 0.0) < 0.5: + return r2 + return evaluate_stage3(program_path) diff --git a/examples/background_blur/fixtures.py b/examples/background_blur/fixtures.py new file mode 100644 index 0000000000..15e4f84692 --- /dev/null +++ b/examples/background_blur/fixtures.py @@ -0,0 +1,235 @@ +""" +Deterministic fixtures, trusted reference implementation, and SSIM. + +Everything here is used ONLY by the evaluator - the evolved program never sees it. +That matters: the golden output is never handed to the candidate, so it cannot be +copied, only reproduced. + +No external data, no webcam, no ML model: the "video" is synthesised from a fixed +seed so the benchmark is byte-for-byte reproducible on any machine. +""" + +import numpy as np + +# Scene / benchmark constants +HEIGHT = 180 +WIDTH = 240 +N_FRAMES = 8 +SIGMA = 4.0 # background blur strength + + +# -------------------------------------------------------------------------- +# Trusted reference (evaluator-side only) +# -------------------------------------------------------------------------- +def gaussian_kernel_1d(sigma: float) -> np.ndarray: + """Truncated (3 sigma) normalised 1D Gaussian, float64.""" + radius = int(np.ceil(3.0 * sigma)) + x = np.arange(-radius, radius + 1, dtype=np.float64) + k = np.exp(-(x**2) / (2.0 * sigma**2)) + return k / k.sum() + + +def reference_blur(image: np.ndarray, sigma: float) -> np.ndarray: + """Exact separable Gaussian blur in float64 with reflect padding. + + This defines the ground truth. The seed program computes the same thing the + slow way (direct 2D convolution), so the seed scores SSIM ~= 1.0 and speedup + 1.0x - it is correct, just slow. + """ + k = gaussian_kernel_1d(sigma) + r = len(k) // 2 + img = image.astype(np.float64) + + # Horizontal pass + padded = np.pad(img, ((0, 0), (r, r), (0, 0)), mode="reflect") + tmp = np.zeros_like(img) + for i, w in enumerate(k): + tmp += w * padded[:, i : i + img.shape[1], :] + + # Vertical pass + padded = np.pad(tmp, ((r, r), (0, 0), (0, 0)), mode="reflect") + out = np.zeros_like(img) + for i, w in enumerate(k): + out += w * padded[i : i + img.shape[0], :, :] + + return out + + +def reference_process(frames, masks, sigma: float): + """Ground-truth portrait blur: sharp person over a blurred background.""" + out = [] + for frame, mask in zip(frames, masks): + blurred = reference_blur(frame, sigma) + m = mask[:, :, None].astype(np.float64) + out.append((m * frame.astype(np.float64) + (1.0 - m) * blurred).astype(np.float32)) + return out + + +def naive_blur(image: np.ndarray, sigma: float) -> np.ndarray: + """The BASELINE to beat: direct 2D convolution, O(k^2) passes per frame. + + This mirrors the seed program. The evaluator keeps its own copy so the + baseline timing never depends on (and cannot be tampered with by) the + candidate under test. + """ + k1 = gaussian_kernel_1d(sigma) + k2 = np.outer(k1, k1) + r = k2.shape[0] // 2 + img = image.astype(np.float64) + padded = np.pad(img, ((r, r), (r, r), (0, 0)), mode="reflect") + out = np.zeros_like(img) + h, w = img.shape[0], img.shape[1] + for dy in range(k2.shape[0]): + for dx in range(k2.shape[1]): + out += k2[dy, dx] * padded[dy : dy + h, dx : dx + w, :] + return out + + +def naive_process(frames, masks, sigma: float): + """Baseline pipeline (what the seed program does).""" + out = [] + for frame, mask in zip(frames, masks): + blurred = naive_blur(frame, sigma) + m = mask[:, :, None].astype(np.float64) + out.append((m * frame.astype(np.float64) + (1.0 - m) * blurred).astype(np.float32)) + return out + + +# -------------------------------------------------------------------------- +# Synthetic "webcam" sequence +# -------------------------------------------------------------------------- +def _background(h: int, w: int, t: float, rng: np.random.Generator) -> np.ndarray: + """A textured background with fine detail (so blur is actually measurable).""" + yy, xx = np.mgrid[0:h, 0:w].astype(np.float32) + + # Gradient + high-frequency texture: detail here is what blurring destroys, + # which is what makes SSIM sensitive to under-blurring. + base = 0.35 + 0.25 * (xx / w) + 0.15 * (yy / h) + texture = 0.10 * np.sin(xx / 3.0) * np.cos(yy / 4.0) + stripes = 0.06 * np.sin((xx + yy) / 2.5) + + img = np.stack( + [ + base + texture + stripes, + base * 0.9 + 0.8 * texture, + base * 0.8 + stripes, + ], + axis=-1, + ) + + # A couple of hard-edged objects (blur must smear these) + img[20:60, 30:80] += 0.20 + img[100:150, 160:220] -= 0.15 + + # Slow global lighting drift across the sequence: the background genuinely + # changes per frame, so "blur once and reuse for every frame" is NOT valid. + img = img * (1.0 + 0.05 * np.sin(t)) + + img += rng.normal(0.0, 0.01, img.shape) + return np.clip(img, 0.0, 1.0).astype(np.float32) + + +def _person_mask(h: int, w: int, cx: float, cy: float) -> np.ndarray: + """Soft-edged head+torso silhouette (values in [0,1], 1 = person).""" + yy, xx = np.mgrid[0:h, 0:w].astype(np.float32) + + def soft_ellipse(ex, ey, rx, ry): + d = ((xx - ex) / rx) ** 2 + ((yy - ey) / ry) ** 2 + # Soft (anti-aliased) edge rather than a hard step + return np.clip(1.5 * (1.0 - d), 0.0, 1.0) + + head = soft_ellipse(cx, cy - 30, 22, 26) + torso = soft_ellipse(cx, cy + 45, 40, 40) + return np.clip(head + torso, 0.0, 1.0).astype(np.float32) + + +def make_sequence(n_frames: int = N_FRAMES, h: int = HEIGHT, w: int = WIDTH): + """Deterministic frames + masks. The person moves; the background drifts.""" + rng = np.random.default_rng(1234) + frames, masks = [], [] + for i in range(n_frames): + t = 2.0 * np.pi * i / max(1, n_frames) + cx = w * 0.5 + 25.0 * np.sin(t) # person walks side to side + cy = h * 0.5 + 6.0 * np.cos(t) + + bg = _background(h, w, t, rng) + mask = _person_mask(h, w, cx, cy) + + # Person appearance: distinctly different from the background + person = np.stack( + [ + np.full((h, w), 0.85, np.float32), + np.full((h, w), 0.55, np.float32), + np.full((h, w), 0.45, np.float32), + ], + axis=-1, + ) + m = mask[:, :, None] + frame = (m * person + (1.0 - m) * bg).astype(np.float32) + + frames.append(frame) + masks.append(mask) + return frames, masks + + +# -------------------------------------------------------------------------- +# SSIM (numpy, no scipy/skimage dependency) +# -------------------------------------------------------------------------- +def _blur64(img2d: np.ndarray, k: np.ndarray) -> np.ndarray: + r = len(k) // 2 + p = np.pad(img2d, ((0, 0), (r, r)), mode="reflect") + tmp = np.zeros_like(img2d) + for i, w in enumerate(k): + tmp += w * p[:, i : i + img2d.shape[1]] + p = np.pad(tmp, ((r, r), (0, 0)), mode="reflect") + out = np.zeros_like(img2d) + for i, w in enumerate(k): + out += w * p[i : i + img2d.shape[0], :] + return out + + +def ssim_map(a: np.ndarray, b: np.ndarray, data_range: float = 1.0) -> np.ndarray: + """Per-pixel SSIM map, averaged over colour channels. Shape (H, W).""" + a = a.astype(np.float64) + b = b.astype(np.float64) + k = gaussian_kernel_1d(1.5) + C1 = (0.01 * data_range) ** 2 + C2 = (0.03 * data_range) ** 2 + + maps = [] + for c in range(a.shape[2]): + x, y = a[:, :, c], b[:, :, c] + mu_x, mu_y = _blur64(x, k), _blur64(y, k) + mu_x2, mu_y2, mu_xy = mu_x * mu_x, mu_y * mu_y, mu_x * mu_y + sx = _blur64(x * x, k) - mu_x2 + sy = _blur64(y * y, k) - mu_y2 + sxy = _blur64(x * y, k) - mu_xy + num = (2 * mu_xy + C1) * (2 * sxy + C2) + den = (mu_x2 + mu_y2 + C1) * (sx + sy + C2) + maps.append(num / den) + return np.mean(maps, axis=0) + + +def ssim(a: np.ndarray, b: np.ndarray, data_range: float = 1.0) -> float: + """Gaussian-windowed SSIM, averaged over colour channels. Range [-1, 1].""" + return float(np.mean(ssim_map(a, b, data_range))) + + +def worst_region_ssim(a: np.ndarray, b: np.ndarray, block: int = 16) -> float: + """SSIM of the WORST block of the image. + + Whole-image mean SSIM hides localised defects: a candidate that leaves a + person-shaped ghost in one corner still averages ~0.99 across the frame. That + is not a hypothetical - the first version of this evaluator scored such a + candidate at 47x and let it through. Grading the worst block instead of the + whole frame is what closes that hole. + """ + m = ssim_map(a, b) + h, w = m.shape + bh, bw = h // block, w // block + if bh == 0 or bw == 0: + return float(np.mean(m)) + # Trim to a whole number of blocks, then average within each block. + m = m[: bh * block, : bw * block] + blocks = m.reshape(bh, block, bw, block).mean(axis=(1, 3)) + return float(np.min(blocks)) diff --git a/examples/background_blur/initial_program.py b/examples/background_blur/initial_program.py new file mode 100644 index 0000000000..6624771e8d --- /dev/null +++ b/examples/background_blur/initial_program.py @@ -0,0 +1,73 @@ +""" +Portrait-mode background blur - the hot function from a video pipeline. + +For every frame we blur the background and composite the (sharp) person back on +top, using a soft segmentation mask that is handed to us: + + blurred = gaussian_blur(frame, sigma) + out = mask * frame + (1 - mask) * blurred + +The seed below is CORRECT but SLOW: it convolves with the full 2D Gaussian +directly, which costs O(k^2) passes per frame. It is the honest, obvious +implementation you would write first - and it is the thing to beat. + +The scoring function does not care how you get there. It only cares that the +output still looks like this one (SSIM gate) and that it is faster. +""" + +import numpy as np + + +# EVOLVE-BLOCK-START +def gaussian_kernel_1d(sigma: float) -> np.ndarray: + """Truncated (3 sigma) normalised 1D Gaussian.""" + radius = int(np.ceil(3.0 * sigma)) + x = np.arange(-radius, radius + 1, dtype=np.float64) + k = np.exp(-(x**2) / (2.0 * sigma**2)) + return k / k.sum() + + +def gaussian_blur(image: np.ndarray, sigma: float) -> np.ndarray: + """Blur an (H, W, 3) image with a Gaussian of the given sigma. + + Naive implementation: builds the full 2D kernel and accumulates one shifted + copy of the image per kernel tap. That is (2*radius+1)^2 passes over the + whole image - correct, but very slow. + """ + k1 = gaussian_kernel_1d(sigma) + k2 = np.outer(k1, k1) # full 2D kernel + r = k2.shape[0] // 2 + + img = image.astype(np.float64) + padded = np.pad(img, ((r, r), (r, r), (0, 0)), mode="reflect") + + out = np.zeros_like(img) + h, w = img.shape[0], img.shape[1] + for dy in range(k2.shape[0]): + for dx in range(k2.shape[1]): + out += k2[dy, dx] * padded[dy : dy + h, dx : dx + w, :] + return out + + +def blur_background(frame: np.ndarray, mask: np.ndarray, sigma: float) -> np.ndarray: + """Blur the background of one frame, keeping the person sharp.""" + blurred = gaussian_blur(frame, sigma) + m = mask[:, :, None].astype(np.float64) + return (m * frame.astype(np.float64) + (1.0 - m) * blurred).astype(np.float32) + + +def process_sequence(frames, masks, sigma: float): + """Process a whole sequence of frames. + + frames: list of (H, W, 3) float32 arrays in [0, 1] + masks: list of (H, W) float32 arrays in [0, 1], 1 = person + returns: list of (H, W, 3) float32 arrays + + The background genuinely changes from frame to frame (the person moves and the + lighting drifts), so results must stay faithful on EVERY frame, not just on + average. + """ + return [blur_background(f, m, sigma) for f, m in zip(frames, masks)] + + +# EVOLVE-BLOCK-END diff --git a/examples/background_blur/test_gaming.py b/examples/background_blur/test_gaming.py new file mode 100644 index 0000000000..d70547e330 --- /dev/null +++ b/examples/background_blur/test_gaming.py @@ -0,0 +1,167 @@ +""" +Adversarial tests for the evaluator - the part people skip. + +The article that inspired this example makes the point well: a lazy candidate that +skips the real work can score beautifully if your scorer lets it. So before you +spend a single LLM token, attack your own evaluator and prove the cheats lose. + +Each test below is a candidate that is FAST but WRONG. All of them must score 0. +Then we prove a genuinely better implementation does win. + + python examples/background_blur/test_gaming.py +""" + +import os +import sys +import tempfile +import unittest + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, HERE) +sys.path.insert(0, os.path.abspath(os.path.join(HERE, "..", ".."))) + +import evaluator as ev # noqa: E402 + +HEADER = "import numpy as np\n" + + +def _write(code): + f = tempfile.NamedTemporaryFile("w", suffix=".py", delete=False) + f.write(HEADER + code) + f.close() + return f.name + + +def _score(code): + path = _write(code) + try: + return ev.evaluate(path) + finally: + os.unlink(path) + + +# -------------------------------------------------------------------------- +# Cheats - all of these are fast, and all of them must be REJECTED +# -------------------------------------------------------------------------- +CHEAT_IDENTITY = """ +def process_sequence(frames, masks, sigma): + # "Optimised": do nothing at all. Blazing fast. + return [f.copy() for f in frames] +""" + +CHEAT_NO_MASK = """ +def process_sequence(frames, masks, sigma): + # Blur everything, including the person - skips the compositing work. + out = [] + for f in frames: + b = f.copy() + for _ in range(3): + b = (b + np.roll(b, 1, 0) + np.roll(b, -1, 0) + np.roll(b, 1, 1) + np.roll(b, -1, 1)) / 5.0 + out.append(b.astype(np.float32)) + return out +""" + +CHEAT_UNDER_BLUR = """ +def process_sequence(frames, masks, sigma): + # Use a much smaller kernel than asked for: far cheaper, visibly wrong. + out = [] + for f, m in zip(frames, masks): + b = f.copy() + b = (b + np.roll(b, 1, 0) + np.roll(b, -1, 0)) / 3.0 + mm = m[:, :, None] + out.append((mm * f + (1 - mm) * b).astype(np.float32)) + return out +""" + +CHEAT_STALE_BACKGROUND = """ +_cache = {} +def process_sequence(frames, masks, sigma): + # Blur frame 0's background once and reuse it for every frame. The background + # actually drifts, so this must fail the worst-frame gate. + from math import ceil + r = int(ceil(3.0 * sigma)) + x = np.arange(-r, r + 1, dtype=np.float64) + k = np.exp(-(x ** 2) / (2 * sigma ** 2)); k /= k.sum() + + def blur(img): + p = np.pad(img, ((0, 0), (r, r), (0, 0)), mode="reflect") + t = np.zeros_like(img) + for i, w in enumerate(k): + t += w * p[:, i:i + img.shape[1], :] + p = np.pad(t, ((r, r), (0, 0), (0, 0)), mode="reflect") + o = np.zeros_like(t) + for i, w in enumerate(k): + o += w * p[i:i + img.shape[0], :, :] + return o + + stale = blur(frames[0].astype(np.float64)) + out = [] + for f, m in zip(frames, masks): + mm = m[:, :, None].astype(np.float64) + out.append((mm * f + (1 - mm) * stale).astype(np.float32)) + return out +""" + +# -------------------------------------------------------------------------- +# A legitimate optimisation - must be ACCEPTED and be much faster +# (separable convolution: 2k passes instead of k^2) +# -------------------------------------------------------------------------- +HONEST_SEPARABLE = """ +def process_sequence(frames, masks, sigma): + r = int(np.ceil(3.0 * sigma)) + x = np.arange(-r, r + 1, dtype=np.float64) + k = np.exp(-(x ** 2) / (2 * sigma ** 2)); k /= k.sum() + + def blur(img): + p = np.pad(img, ((0, 0), (r, r), (0, 0)), mode="reflect") + t = np.zeros_like(img) + for i, w in enumerate(k): + t += w * p[:, i:i + img.shape[1], :] + p = np.pad(t, ((r, r), (0, 0), (0, 0)), mode="reflect") + o = np.zeros_like(t) + for i, w in enumerate(k): + o += w * p[i:i + img.shape[0], :, :] + return o + + out = [] + for f, m in zip(frames, masks): + img = f.astype(np.float64) + b = blur(img) + mm = m[:, :, None].astype(np.float64) + out.append((mm * img + (1 - mm) * b).astype(np.float32)) + return out +""" + + +class TestEvaluatorCannotBeGamed(unittest.TestCase): + def _assert_rejected(self, code, label): + res = _score(code) + score = res.metrics.get("combined_score", 0.0) + self.assertEqual(score, 0.0, f"{label} should be REJECTED but scored {score}") + # A rejection must always explain itself. + self.assertIn("failure_reason", res.artifacts, f"{label}: no failure_reason artifact") + print(f" [rejected] {label}: {res.artifacts['failure_reason'][:90]}") + + def test_identity_is_rejected(self): + self._assert_rejected(CHEAT_IDENTITY, "do-nothing (returns input)") + + def test_blurring_the_person_is_rejected(self): + self._assert_rejected(CHEAT_NO_MASK, "blurs everything (ignores mask)") + + def test_under_blur_is_rejected(self): + self._assert_rejected(CHEAT_UNDER_BLUR, "under-blurs (kernel far too small)") + + def test_stale_background_is_rejected(self): + self._assert_rejected(CHEAT_STALE_BACKGROUND, "reuses frame 0's background") + + def test_honest_optimisation_wins(self): + res = _score(HONEST_SEPARABLE) + speedup = res.metrics.get("combined_score", 0.0) + ssim = res.metrics.get("ssim", 0.0) + self.assertGreater(speedup, 1.0, "separable blur should beat the naive baseline") + self.assertGreaterEqual(ssim, ev.MEAN_SSIM_GATE, "and it should pass the quality gate") + print(f" [accepted] separable convolution: {speedup:.2f}x speedup, SSIM {ssim:.4f}") + + +if __name__ == "__main__": + unittest.main(verbosity=2) From 5cb7b15a6a83497fdda8c9c8f4cb1e6f4fd1c3a7 Mon Sep 17 00:00:00 2001 From: Asankhaya Sharma Date: Tue, 14 Jul 2026 08:08:02 +0800 Subject: [PATCH 3/3] Propagate llm.provider to per-model configs (#472); add background_blur example; bump 0.3.1 Provider propagation fix: - LLMConfig pushes shared settings down to each LLMModelConfig via a `shared_config` dict, but that dict omitted `provider`. Every model therefore kept provider=None, and LLMEnsemble._create_model (which routes on `provider`) sent them all to the OpenAI backend regardless of the config. A config with `provider: claude_code` would crash with a missing-OPENAI_API_KEY error, making the examples/claude_code_quickstart example unusable. - Add `provider` to both shared_config dicts (__post_init__ and rebuild_models). Propagation uses overwrite=False, so an explicit per-model provider still wins. - Add tests/test_provider_propagation.py: covers propagation to evolution and evaluator models, per-model override precedence, the unchanged None default, survival across rebuild_models(), and end-to-end that LLMEnsemble now actually constructs ClaudeCodeLLM. All four propagation assertions fail without the fix. Independently reimplemented; supersedes #472. New example - examples/background_blur: Evolving a hot function behind a hard quality gate. The score is a speedup, but fidelity is pass/fail, so a fast-but-wrong candidate scores zero. Deterministic and numpy-only (the video is synthesised from a fixed seed - no webcam, GPU or dataset). Demonstrates the cascade (cheap smoke -> quality gate -> benchmark, so timing is only spent on candidates that are already correct), artifacts that tell the model WHY it was rejected, and a (complexity, ssim) MAP-Elites grid. Two traps it documents, both found the hard way: - The obvious gate is exploitable. Grading mean and worst-FRAME SSIM lets a "blur frame 0's background and reuse it forever" candidate score 47x while leaving a visible person-shaped ghost - it damages one region and the frame average hides it. Grading the worst 16x16 REGION catches it. test_gaming.py encodes the cheats as tests so they must lose. - Timing-as-fitness needs care: caching a single baseline measurement lets a transiently loaded machine inflate every later speedup. Baseline and candidate are now measured interleaved, warmed up, best-of-N. Bump version to 0.3.1. Co-Authored-By: Claude Opus 4.8 (1M context) --- examples/background_blur/README.md | 5 +- examples/background_blur/config.yaml | 10 ++-- openevolve/_version.py | 2 +- openevolve/config.py | 9 +++ tests/test_provider_propagation.py | 83 ++++++++++++++++++++++++++++ 5 files changed, 100 insertions(+), 9 deletions(-) create mode 100644 tests/test_provider_propagation.py diff --git a/examples/background_blur/README.md b/examples/background_blur/README.md index 571926a1e1..eb4f0c9782 100644 --- a/examples/background_blur/README.md +++ b/examples/background_blur/README.md @@ -37,7 +37,7 @@ would write first, and it is the thing to beat. ## Run it ```bash -export OPENROUTER_API_KEY=sk-or-... +export OPENAI_API_KEY=sk-or-... # your OpenRouter key python openevolve-run.py \ examples/background_blur/initial_program.py \ @@ -46,7 +46,8 @@ python openevolve-run.py \ --iterations 100 ``` -`config.yaml` uses OpenRouter with a cheap model (`google/gemini-2.5-flash-lite`). +`config.yaml` uses OpenRouter with a cheap model (`google/gemini-2.5-flash-lite`); the +key is read from `OPENAI_API_KEY`. Any OpenAI-compatible endpoint works — point `api_base` at a local optillm server to run with no hosted API and no key at all. diff --git a/examples/background_blur/config.yaml b/examples/background_blur/config.yaml index 3e022ad338..17fbfd6655 100644 --- a/examples/background_blur/config.yaml +++ b/examples/background_blur/config.yaml @@ -1,14 +1,14 @@ # Portrait background blur - optimise a hot function behind a hard quality gate. # -# Uses OpenRouter with a cheap, fast model. Set your key first: +# Uses OpenRouter with a cheap, fast model. Put your OpenRouter key in the usual +# environment variable before running: # -# export OPENROUTER_API_KEY=sk-or-... +# export OPENAI_API_KEY=sk-or-... # # Any OpenAI-compatible endpoint works. To run fully locally with no hosted API and -# no key, point `api_base` at a local optillm server instead, e.g.: +# no key at all, point `api_base` at a local optillm server instead, e.g.: # # api_base: "http://localhost:8000/v1" -# api_key: "dummy" # models: [{ name: "optillm", api_base: "http://localhost:8000/v1", weight: 1.0 }] max_iterations: 100 @@ -19,7 +19,6 @@ max_code_length: 40000 llm: api_base: "https://openrouter.ai/api/v1" - api_key: "${OPENROUTER_API_KEY}" max_tokens: 6000 temperature: 0.9 timeout: 600 @@ -28,7 +27,6 @@ llm: models: - name: "google/gemini-2.5-flash-lite" api_base: "https://openrouter.ai/api/v1" - api_key: "${OPENROUTER_API_KEY}" weight: 1.0 prompt: diff --git a/openevolve/_version.py b/openevolve/_version.py index 88e72b8058..179eae7e3b 100644 --- a/openevolve/_version.py +++ b/openevolve/_version.py @@ -1,3 +1,3 @@ """Version information for openevolve package.""" -__version__ = "0.3.0" +__version__ = "0.3.1" diff --git a/openevolve/config.py b/openevolve/config.py index fd03ed7a98..c19ab4ca1d 100644 --- a/openevolve/config.py +++ b/openevolve/config.py @@ -174,6 +174,12 @@ def __post_init__(self): # Update models with shared configuration values shared_config = { + # `provider` must be shared: without it a top-level `llm.provider` + # (e.g. "claude_code") never reaches the per-model configs, every model + # keeps provider=None, and LLMEnsemble silently routes them all to the + # OpenAI backend. Propagation uses overwrite=False, so an explicit + # per-model provider still wins. + "provider": self.provider, "api_base": self.api_base, "api_key": self.api_key, "temperature": self.temperature, @@ -228,6 +234,9 @@ def rebuild_models(self) -> None: # Update models with shared configuration values shared_config = { + # See the note in __post_init__: `provider` has to be propagated here too, + # or rebuilding the models drops a top-level provider back to None. + "provider": self.provider, "api_base": self.api_base, "api_key": self.api_key, "temperature": self.temperature, diff --git a/tests/test_provider_propagation.py b/tests/test_provider_propagation.py new file mode 100644 index 0000000000..cd6cdbf415 --- /dev/null +++ b/tests/test_provider_propagation.py @@ -0,0 +1,83 @@ +""" +Tests that a top-level `llm.provider` reaches the per-model configs. + +Regression test: LLMConfig pushes shared settings down to each LLMModelConfig via a +`shared_config` dict, but that dict omitted `provider`. Every model therefore kept +`provider=None`, and LLMEnsemble._create_model (which routes on `provider`) sent them +all to the OpenAI backend regardless of what the config asked for. A config with +`provider: claude_code` would crash with a missing-OPENAI_API_KEY error. + +Reported in https://github.com/algorithmicsuperintelligence/openevolve/pull/472 +""" + +import os +import unittest + +os.environ.setdefault("OPENAI_API_KEY", "test") + +from openevolve.config import Config, LLMConfig + + +class TestProviderPropagation(unittest.TestCase): + def test_top_level_provider_reaches_evolution_models(self): + config = Config.from_dict( + {"llm": {"provider": "claude_code", "models": [{"name": "a"}, {"name": "b"}]}} + ) + self.assertEqual([m.provider for m in config.llm.models], ["claude_code", "claude_code"]) + + def test_top_level_provider_reaches_evaluator_models(self): + config = Config.from_dict( + {"llm": {"provider": "claude_code", "models": [{"name": "a"}, {"name": "b"}]}} + ) + self.assertEqual( + [m.provider for m in config.llm.evaluator_models], ["claude_code", "claude_code"] + ) + + def test_explicit_per_model_provider_wins(self): + """Propagation must not clobber a provider set explicitly on a model.""" + config = Config.from_dict( + { + "llm": { + "provider": "claude_code", + "models": [{"name": "a", "provider": "openai"}, {"name": "b"}], + } + } + ) + self.assertEqual([m.provider for m in config.llm.models], ["openai", "claude_code"]) + + def test_default_provider_is_none(self): + """No provider configured -> still None, i.e. the OpenAI default path.""" + config = Config.from_dict({"llm": {"models": [{"name": "a"}]}}) + self.assertEqual([m.provider for m in config.llm.models], [None]) + + def test_provider_survives_rebuild_models(self): + """rebuild_models() re-derives the shared config; it must not drop provider.""" + llm = LLMConfig(provider="claude_code") + llm.rebuild_models() + for model in llm.models + llm.evaluator_models: + self.assertEqual(model.provider, "claude_code") + + +class TestEnsembleRoutesOnProvider(unittest.TestCase): + """End-to-end: the ensemble must actually construct the provider's backend.""" + + def test_ensemble_builds_provider_backend(self): + from openevolve.llm.ensemble import LLMEnsemble + + config = Config.from_dict( + {"llm": {"provider": "claude_code", "models": [{"name": "sonnet"}]}} + ) + ensemble = LLMEnsemble(config.llm.models) + # Before the fix this silently produced an OpenAILLM. + self.assertEqual(type(ensemble.models[0]).__name__, "ClaudeCodeLLM") + + def test_ensemble_defaults_to_openai_without_provider(self): + from openevolve.llm.ensemble import LLMEnsemble + + config = Config.from_dict({"llm": {"models": [{"name": "gpt-4"}]}}) + ensemble = LLMEnsemble(config.llm.models) + self.assertEqual(type(ensemble.models[0]).__name__, "OpenAILLM") + + +if __name__ == "__main__": + unittest.main()