fuel-alert/app/Services/Forecasting/BacktestRunner.php

<?php

namespace App\Services\Forecasting;

use App\Models\Backtest;
use App\Services\Forecasting\Contracts\WeeklyForecastModel;
use Carbon\CarbonInterface;
use Illuminate\Support\Facades\DB;

/**
 * Runs a WeeklyForecastModel through a train/eval split and persists
 * the result to the `backtests` table.
 *
 * Pipeline:
 *   1. Generate the training and eval Monday lists from the date ranges.
 *   2. Run LeakDetector against every Monday × every feature. Refuse to
 *      train if any source date is on or after a target Monday.
 *   3. Train the model.
 *   4. For each eval Monday: predict, look up actual ΔULSP from
 *      `weekly_pump_prices`, score directional accuracy + abs error.
 *   5. Persist a Backtest row, return it.
 *
 * The `leak_suspected` flag is a *secondary* smell test (true when
 * directional_accuracy > 75). Primary leak defence is step 2.
 */
final class BacktestRunner
{
    private const float FLAT_THRESHOLD_PENCE_X100 = 20.0; // 0.2 p/L

    public function __construct(
        private readonly LeakDetector $leakDetector = new LeakDetector,
    ) {}

    public function run(
        WeeklyForecastModel $model,
        CarbonInterface $trainStart,
        CarbonInterface $trainEnd,
        CarbonInterface $evalStart,
        CarbonInterface $evalEnd,
    ): Backtest {
        $trainingMondays = $this->mondaysBetween($trainStart, $trainEnd);
        $evalMondays = $this->mondaysBetween($evalStart, $evalEnd);

        $spec = $model->featureSpec();
        $report = $this->leakDetector->validate($spec, [...$trainingMondays, ...$evalMondays]);
        if ($report->hasLeaks()) {
            throw new LeakDetectorException($report);
        }

        $model->train($trainingMondays);

        $correct = 0;
        $totalScored = 0;
        $absErrors = [];
        $bins = [];

        foreach ($evalMondays as $monday) {
            $actualDelta = $this->actualDeltaPence($monday);
            if ($actualDelta === null) {
                continue;
            }

            $prediction = $model->predict($monday);
            $actualDirection = $this->classifyDirection($actualDelta);
            $hit = $prediction->direction === $actualDirection;

            $totalScored++;
            $absErrors[] = abs($prediction->magnitudePence - $actualDelta);
            if ($hit) {
                $correct++;
            }

            $bin = $this->bucketForMagnitude($prediction->magnitudePence);
            $bins[$bin] ??= ['correct' => 0, 'total' => 0];
            $bins[$bin]['total']++;
            if ($hit) {
                $bins[$bin]['correct']++;
            }
        }

        $directionalAccuracy = $totalScored === 0
            ? null
            : round(($correct / $totalScored) * 100, 2);

        $maePence = $absErrors === []
            ? null
            : round((array_sum($absErrors) / count($absErrors)) / 100, 2);

        $calibrationTable = [];
        foreach ($bins as $key => $b) {
            $calibrationTable[$key] = round($b['correct'] / $b['total'], 4);
        }

        return Backtest::create([
            'model_version' => $spec->modelVersion(),
            'features_json' => $spec->toArray(),
            'coefficients_json' => $model->coefficients(),
            'train_start' => $trainStart->toDateString(),
            'train_end' => $trainEnd->toDateString(),
            'eval_start' => $evalStart->toDateString(),
            'eval_end' => $evalEnd->toDateString(),
            'directional_accuracy' => $directionalAccuracy,
            'mae_pence' => $maePence,
            'calibration_table' => $calibrationTable,
            'leak_suspected' => $directionalAccuracy !== null && $directionalAccuracy > 75.0,
            'ran_at' => now(),
        ]);
    }

    /** @return array<int, CarbonInterface> */
    private function mondaysBetween(CarbonInterface $start, CarbonInterface $end): array
    {
        $mondays = [];
        $cursor = $start->copy()->startOfDay();
        $boundary = $end->copy()->startOfDay();

        while ($cursor->lessThanOrEqualTo($boundary)) {
            if ($cursor->dayOfWeek === CarbonInterface::MONDAY) {
                $mondays[] = $cursor->copy();
            }
            $cursor = $cursor->addDay();
        }

        return $mondays;
    }

    private function actualDeltaPence(CarbonInterface $targetMonday): ?float
    {
        $current = DB::table('weekly_pump_prices')
            ->where('date', $targetMonday->toDateString())
            ->value('ulsp_pence');
        $previous = DB::table('weekly_pump_prices')
            ->where('date', $targetMonday->copy()->subDays(7)->toDateString())
            ->value('ulsp_pence');

        if ($current === null || $previous === null) {
            return null;
        }

        return (float) ($current - $previous);
    }

    private function classifyDirection(float $deltaPence): string
    {
        return match (true) {
            $deltaPence > self::FLAT_THRESHOLD_PENCE_X100 => 'rising',
            $deltaPence < -self::FLAT_THRESHOLD_PENCE_X100 => 'falling',
            default => 'flat',
        };
    }

    private function bucketForMagnitude(float $magnitudePence): string
    {
        $abs = abs($magnitudePence);

        return match (true) {
            $abs < 50.0 => '0.0-0.5p',
            $abs < 100.0 => '0.5-1.0p',
            default => '1.0p+',
        };
    }
}