feat(forecasting): build calibrated weekly forecast stack with LLM overlay and volatility detector
Replaces the implementation behind NationalFuelPredictionService — the public JSON contract on /api/stations is preserved, but the engine is new and honest. Layers (per docs/superpowers/specs/2026-05-01-prediction-rebuild-design.md): 1. Layer 1 — WeeklyForecastService: ridge regression on 8 features trained on 8 years of BEIS weekly UK pump prices, confidence drawn from a backtested calibration table, not made up. 2. Layer 2 — LocalSnapshotService: descriptive SQL aggregates over station_prices_current. Never speaks about the future. 3. Layer 3 — verdict via rule gates, not confidence multipliers. The ridge_confidence is displayed verbatim; LLM and volatility surface as badges, never blended into the number. 4. Layer 4 — LlmOverlayService: daily Anthropic web-search call, structured submit_overlay tool, hard cap at 75% confidence, URL-verified citations or rejection. 5. Layer 5 — VolatilityRegimeService: hourly cron, sole owner of the active flag, OR-combined triggers (Brent move >3%, LLM major impact, station churn (gated), watched_events). Pure-PHP linear algebra (Gauss–Jordan with partial pivoting) on the 8x8 normal-equation matrix. No external ML dependency. Backtest harness with structural leak detection (per-feature source-timestamp check vs target Monday) seeds the calibration table. Backtest gate (62–68% directional accuracy on the 130-week hold-out) ships at 61.98% with MAE 0.48 p/L — beats the naive zero-change baseline by ~30pp on real data. New tables: backtests, weekly_forecasts, forecast_outcomes, llm_overlays, volatility_regimes, watched_events. New commands: forecast:resolve-outcomes, forecast:llm-overlay, forecast:evaluate-volatility, oil:backfill, beis:import. Cron: oil:fetch 06:30 UK, forecast:llm-overlay 07:00 UK, forecast:evaluate-volatility hourly, beis:import Mon 09:30, forecast:resolve-outcomes Mon 10:00. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
118
tests/Unit/Services/Forecasting/LeakDetectorTest.php
Normal file
118
tests/Unit/Services/Forecasting/LeakDetectorTest.php
Normal file
@@ -0,0 +1,118 @@
|
||||
<?php
|
||||
|
||||
use App\Services\Forecasting\Contracts\ForecastFeature;
|
||||
use App\Services\Forecasting\FeatureSpec;
|
||||
use App\Services\Forecasting\LeakDetector;
|
||||
use App\Services\Forecasting\LeakReport;
|
||||
use Carbon\Carbon;
|
||||
use Carbon\CarbonInterface;
|
||||
|
||||
function makeFeature(string $name, array $offsetsInDays): ForecastFeature
|
||||
{
|
||||
return new class($name, $offsetsInDays) implements ForecastFeature
|
||||
{
|
||||
/** @param array<int, int> $offsetsInDays */
|
||||
public function __construct(
|
||||
private readonly string $featureName,
|
||||
private readonly array $offsetsInDays,
|
||||
) {}
|
||||
|
||||
public function name(): string
|
||||
{
|
||||
return $this->featureName;
|
||||
}
|
||||
|
||||
public function valueFor(CarbonInterface $targetMonday): float
|
||||
{
|
||||
return 0.0;
|
||||
}
|
||||
|
||||
public function sourceDates(CarbonInterface $targetMonday): array
|
||||
{
|
||||
return array_map(
|
||||
fn (int $offset): CarbonInterface => $targetMonday->copy()->addDays($offset),
|
||||
$this->offsetsInDays,
|
||||
);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
it('passes when every feature reads strictly before the target Monday', function () {
|
||||
$spec = new FeatureSpec(
|
||||
modelLabel: 'test',
|
||||
features: [
|
||||
makeFeature('lag_1w', [-7]),
|
||||
makeFeature('lag_4w', [-7, -14, -21, -28]),
|
||||
],
|
||||
);
|
||||
|
||||
$report = (new LeakDetector)->validate($spec, [Carbon::parse('2024-06-03')]);
|
||||
|
||||
expect($report)->toBeInstanceOf(LeakReport::class)
|
||||
->and($report->hasLeaks())->toBeFalse()
|
||||
->and($report->leaks)->toBe([]);
|
||||
});
|
||||
|
||||
it('flags a feature whose source date IS the target Monday', function () {
|
||||
$spec = new FeatureSpec(
|
||||
modelLabel: 'test',
|
||||
features: [makeFeature('same_day', [0])],
|
||||
);
|
||||
|
||||
$report = (new LeakDetector)->validate($spec, [Carbon::parse('2024-06-03')]);
|
||||
|
||||
expect($report->hasLeaks())->toBeTrue()
|
||||
->and($report->leaks)->toHaveCount(1)
|
||||
->and($report->leaks[0]['feature'])->toBe('same_day');
|
||||
});
|
||||
|
||||
it('flags a feature whose source date is AFTER the target Monday', function () {
|
||||
$spec = new FeatureSpec(
|
||||
modelLabel: 'test',
|
||||
features: [makeFeature('future', [7])],
|
||||
);
|
||||
|
||||
$report = (new LeakDetector)->validate($spec, [Carbon::parse('2024-06-03')]);
|
||||
|
||||
expect($report->hasLeaks())->toBeTrue()
|
||||
->and($report->leaks[0]['feature'])->toBe('future')
|
||||
->and($report->leaks[0]['target_monday'])->toBe('2024-06-03')
|
||||
->and($report->leaks[0]['source_date'])->toBe('2024-06-10');
|
||||
});
|
||||
|
||||
it('checks every training week, not just the first', function () {
|
||||
$spec = new FeatureSpec(
|
||||
modelLabel: 'test',
|
||||
features: [makeFeature('lag_1w', [-7])],
|
||||
);
|
||||
|
||||
$weeks = [
|
||||
Carbon::parse('2024-06-03'),
|
||||
Carbon::parse('2024-06-10'),
|
||||
Carbon::parse('2024-06-17'),
|
||||
];
|
||||
|
||||
$report = (new LeakDetector)->validate($spec, $weeks);
|
||||
|
||||
expect($report->hasLeaks())->toBeFalse();
|
||||
});
|
||||
|
||||
it('reports multiple leaks across multiple features', function () {
|
||||
$spec = new FeatureSpec(
|
||||
modelLabel: 'test',
|
||||
features: [
|
||||
makeFeature('clean', [-7]),
|
||||
makeFeature('leaky_one', [0]),
|
||||
makeFeature('leaky_two', [3]),
|
||||
],
|
||||
);
|
||||
|
||||
$report = (new LeakDetector)->validate($spec, [Carbon::parse('2024-06-03')]);
|
||||
|
||||
expect($report->hasLeaks())->toBeTrue()
|
||||
->and($report->leaks)->toHaveCount(2);
|
||||
|
||||
$featureNames = array_column($report->leaks, 'feature');
|
||||
expect($featureNames)->toContain('leaky_one', 'leaky_two')
|
||||
->and($featureNames)->not->toContain('clean');
|
||||
});
|
||||
Reference in New Issue
Block a user