Files
fuel-alert/tests/Unit/Services/Forecasting/Phase6Test.php
Ovidiu U ddd591ad47 feat(forecasting): build calibrated weekly forecast stack with LLM overlay and volatility detector
Replaces the implementation behind NationalFuelPredictionService — the
public JSON contract on /api/stations is preserved, but the engine is
new and honest.

Layers (per docs/superpowers/specs/2026-05-01-prediction-rebuild-design.md):
1. Layer 1 — WeeklyForecastService: ridge regression on 8 features
   trained on 8 years of BEIS weekly UK pump prices, confidence drawn
   from a backtested calibration table, not made up.
2. Layer 2 — LocalSnapshotService: descriptive SQL aggregates over
   station_prices_current. Never speaks about the future.
3. Layer 3 — verdict via rule gates, not confidence multipliers. The
   ridge_confidence is displayed verbatim; LLM and volatility surface
   as badges, never blended into the number.
4. Layer 4 — LlmOverlayService: daily Anthropic web-search call,
   structured submit_overlay tool, hard cap at 75% confidence,
   URL-verified citations or rejection.
5. Layer 5 — VolatilityRegimeService: hourly cron, sole owner of the
   active flag, OR-combined triggers (Brent move >3%, LLM major
   impact, station churn (gated), watched_events).

Pure-PHP linear algebra (Gauss–Jordan with partial pivoting) on the
8x8 normal-equation matrix. No external ML dependency. Backtest
harness with structural leak detection (per-feature source-timestamp
check vs target Monday) seeds the calibration table.

Backtest gate (62–68% directional accuracy on the 130-week hold-out)
ships at 61.98% with MAE 0.48 p/L — beats the naive zero-change
baseline by ~30pp on real data.

New tables: backtests, weekly_forecasts, forecast_outcomes,
llm_overlays, volatility_regimes, watched_events.

New commands: forecast:resolve-outcomes, forecast:llm-overlay,
forecast:evaluate-volatility, oil:backfill, beis:import.

Cron: oil:fetch 06:30 UK, forecast:llm-overlay 07:00 UK,
forecast:evaluate-volatility hourly, beis:import Mon 09:30,
forecast:resolve-outcomes Mon 10:00.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-03 08:40:05 +01:00

167 lines
5.4 KiB
PHP

<?php
use App\Services\Forecasting\AccuracyHistory;
use App\Services\Forecasting\DutyChangeDetector;
use App\Services\Forecasting\OutcomeResolver;
use Carbon\Carbon;
use Illuminate\Foundation\Testing\RefreshDatabase;
use Illuminate\Support\Facades\DB;
uses(RefreshDatabase::class);
function seedFlatPrices(int $weeks, int $duty = 5295): void
{
$start = Carbon::parse('2024-01-01');
for ($i = 0; $i < $weeks; $i++) {
DB::table('weekly_pump_prices')->insert([
'date' => $start->copy()->addWeeks($i)->toDateString(),
'ulsp_pence' => 14000,
'ulsd_pence' => 15000,
'ulsp_duty_pence' => $duty,
'ulsd_duty_pence' => $duty,
'ulsp_vat_pct' => 20,
'ulsd_vat_pct' => 20,
]);
}
}
it('DutyChangeDetector returns false when duty is constant across the window', function () {
seedFlatPrices(20);
$detector = new DutyChangeDetector;
expect($detector->isAdjacent(Carbon::parse('2024-03-04')))->toBeFalse();
});
it('DutyChangeDetector returns true when duty changes within ±4 weeks', function () {
// 8 weeks at 57.95p, then 8 weeks at 52.95p
$start = Carbon::parse('2024-01-01');
for ($i = 0; $i < 8; $i++) {
DB::table('weekly_pump_prices')->insert([
'date' => $start->copy()->addWeeks($i)->toDateString(),
'ulsp_pence' => 14000,
'ulsd_pence' => 15000,
'ulsp_duty_pence' => 5795,
'ulsd_duty_pence' => 5795,
'ulsp_vat_pct' => 20,
'ulsd_vat_pct' => 20,
]);
}
for ($i = 8; $i < 16; $i++) {
DB::table('weekly_pump_prices')->insert([
'date' => $start->copy()->addWeeks($i)->toDateString(),
'ulsp_pence' => 14000,
'ulsd_pence' => 15000,
'ulsp_duty_pence' => 5295,
'ulsd_duty_pence' => 5295,
'ulsp_vat_pct' => 20,
'ulsd_vat_pct' => 20,
]);
}
$detector = new DutyChangeDetector;
// Target Mon at week 7 — change happens at week 8 → within ±4 weeks
expect($detector->isAdjacent(Carbon::parse('2024-02-19')))->toBeTrue();
});
it('AccuracyHistory returns null when fewer than 4 outcomes', function () {
$history = new AccuracyHistory;
DB::table('forecast_outcomes')->insert([
'forecast_for' => Carbon::now()->subWeeks(2)->toDateString(),
'model_version' => 'm1',
'predicted_class' => 'rising',
'actual_class' => 'rising',
'correct' => true,
'abs_error_pence' => 50,
'resolved_at' => now(),
]);
expect($history->trailingHitRate('m1'))->toBeNull();
});
it('AccuracyHistory computes hit rate over the last 13 weeks', function () {
$history = new AccuracyHistory;
// 4 correct, 1 wrong → 80%
foreach ([true, true, true, true, false] as $i => $correct) {
DB::table('forecast_outcomes')->insert([
'forecast_for' => Carbon::now()->subWeeks($i + 1)->toDateString(),
'model_version' => 'm1',
'predicted_class' => 'rising',
'actual_class' => $correct ? 'rising' : 'falling',
'correct' => $correct,
'abs_error_pence' => 50,
'resolved_at' => now(),
]);
}
expect($history->trailingHitRate('m1'))->toBe(0.8);
});
it('AccuracyHistory excludes outcomes outside the 13-week window', function () {
$history = new AccuracyHistory;
// 4 inside window (correct), 4 outside (wrong) → 100% inside
foreach (range(1, 4) as $i) {
DB::table('forecast_outcomes')->insert([
'forecast_for' => Carbon::now()->subWeeks($i)->toDateString(),
'model_version' => 'm1',
'predicted_class' => 'rising',
'actual_class' => 'rising',
'correct' => true,
'abs_error_pence' => 0,
'resolved_at' => now(),
]);
}
foreach (range(20, 23) as $i) {
DB::table('forecast_outcomes')->insert([
'forecast_for' => Carbon::now()->subWeeks($i)->toDateString(),
'model_version' => 'm1',
'predicted_class' => 'rising',
'actual_class' => 'falling',
'correct' => false,
'abs_error_pence' => 100,
'resolved_at' => now(),
]);
}
expect($history->trailingHitRate('m1'))->toBe(1.0);
});
it('OutcomeResolver pairs forecasts with actual deltas idempotently', function () {
seedFlatPrices(20);
// Insert a forecast for week index 5 (2024-02-05)
DB::table('weekly_forecasts')->insert([
'forecast_for' => '2024-02-05',
'model_version' => 'ridge-test',
'direction' => 'rising',
'magnitude_pence' => 80,
'ridge_confidence' => 60,
'flagged_duty_change' => false,
'reasoning' => 'test',
'generated_at' => now(),
'created_at' => now(),
'updated_at' => now(),
]);
// Override now() so the resolver sees the forecast as past.
Carbon::setTestNow('2024-02-12');
$resolver = new OutcomeResolver;
$first = $resolver->resolvePending();
$second = $resolver->resolvePending();
expect($first)->toBe(1)
->and($second)->toBe(0); // idempotent on re-run
$row = DB::table('forecast_outcomes')->where('forecast_for', '2024-02-05')->first();
expect($row->predicted_class)->toBe('rising')
->and($row->actual_class)->toBe('flat') // flat data → actual delta = 0
->and((bool) $row->correct)->toBeFalse();
Carbon::setTestNow();
});