75). Primary leak defence is step 2. */ final class BacktestRunner { private const float FLAT_THRESHOLD_PENCE_X100 = 20.0; // 0.2 p/L public function __construct( private readonly LeakDetector $leakDetector = new LeakDetector, ) {} public function run( WeeklyForecastModel $model, CarbonInterface $trainStart, CarbonInterface $trainEnd, CarbonInterface $evalStart, CarbonInterface $evalEnd, ): Backtest { $trainingMondays = $this->mondaysBetween($trainStart, $trainEnd); $evalMondays = $this->mondaysBetween($evalStart, $evalEnd); $spec = $model->featureSpec(); $report = $this->leakDetector->validate($spec, [...$trainingMondays, ...$evalMondays]); if ($report->hasLeaks()) { throw new LeakDetectorException($report); } $model->train($trainingMondays); $correct = 0; $totalScored = 0; $absErrors = []; $bins = []; foreach ($evalMondays as $monday) { $actualDelta = $this->actualDeltaPence($monday); if ($actualDelta === null) { continue; } $prediction = $model->predict($monday); $actualDirection = $this->classifyDirection($actualDelta); $hit = $prediction->direction === $actualDirection; $totalScored++; $absErrors[] = abs($prediction->magnitudePence - $actualDelta); if ($hit) { $correct++; } $bin = $this->bucketForMagnitude($prediction->magnitudePence); $bins[$bin] ??= ['correct' => 0, 'total' => 0]; $bins[$bin]['total']++; if ($hit) { $bins[$bin]['correct']++; } } $directionalAccuracy = $totalScored === 0 ? null : round(($correct / $totalScored) * 100, 2); $maePence = $absErrors === [] ? null : round((array_sum($absErrors) / count($absErrors)) / 100, 2); $calibrationTable = []; foreach ($bins as $key => $b) { $calibrationTable[$key] = round($b['correct'] / $b['total'], 4); } return Backtest::create([ 'model_version' => $spec->modelVersion(), 'features_json' => $spec->toArray(), 'coefficients_json' => $model->coefficients(), 'train_start' => $trainStart->toDateString(), 'train_end' => $trainEnd->toDateString(), 'eval_start' => $evalStart->toDateString(), 'eval_end' => $evalEnd->toDateString(), 'directional_accuracy' => $directionalAccuracy, 'mae_pence' => $maePence, 'calibration_table' => $calibrationTable, 'leak_suspected' => $directionalAccuracy !== null && $directionalAccuracy > 75.0, 'ran_at' => now(), ]); } /** @return array */ private function mondaysBetween(CarbonInterface $start, CarbonInterface $end): array { $mondays = []; $cursor = $start->copy()->startOfDay(); $boundary = $end->copy()->startOfDay(); while ($cursor->lessThanOrEqualTo($boundary)) { if ($cursor->dayOfWeek === CarbonInterface::MONDAY) { $mondays[] = $cursor->copy(); } $cursor = $cursor->addDay(); } return $mondays; } private function actualDeltaPence(CarbonInterface $targetMonday): ?float { $current = DB::table('weekly_pump_prices') ->where('date', $targetMonday->toDateString()) ->value('ulsp_pence'); $previous = DB::table('weekly_pump_prices') ->where('date', $targetMonday->copy()->subDays(7)->toDateString()) ->value('ulsp_pence'); if ($current === null || $previous === null) { return null; } return (float) ($current - $previous); } private function classifyDirection(float $deltaPence): string { return match (true) { $deltaPence > self::FLAT_THRESHOLD_PENCE_X100 => 'rising', $deltaPence < -self::FLAT_THRESHOLD_PENCE_X100 => 'falling', default => 'flat', }; } private function bucketForMagnitude(float $magnitudePence): string { $abs = abs($magnitudePence); return match (true) { $abs < 50.0 => '0.0-0.5p', $abs < 100.0 => '0.5-1.0p', default => '1.0p+', }; } }