feat: add LLM prediction providers with structured output support
Some checks failed
linter / quality (push) Has been cancelled
tests / ci (8.3) (push) Has been cancelled
tests / ci (8.4) (push) Has been cancelled
tests / ci (8.5) (push) Has been cancelled

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Ovidiu U
2026-04-07 14:42:44 +01:00
parent e9612666e3
commit 6a80c11f38
18 changed files with 1101 additions and 484 deletions

View File

@@ -0,0 +1,219 @@
<?php
use App\Enums\PredictionSource;
use App\Enums\TrendDirection;
use App\Models\BrentPrice;
use App\Services\ApiLogger;
use App\Services\LlmPrediction\AnthropicPredictionProvider;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Http;
beforeEach(function (): void {
Http::preventStrayRequests();
config(['services.anthropic.api_key' => 'test-key']);
$this->provider = new AnthropicPredictionProvider(new ApiLogger);
});
it('returns null when api key is not configured', function (): void {
config(['services.anthropic.api_key' => null]);
$prices = fakePrices(14);
expect($this->provider->predict($prices))->toBeNull();
});
it('uses submit_prediction tool in the basic request', function (): void {
Http::fake([
'https://api.anthropic.com/*' => Http::response([
'stop_reason' => 'tool_use',
'content' => [[
'type' => 'tool_use',
'name' => 'submit_prediction',
'input' => ['direction' => 'rising', 'confidence' => 72, 'reasoning' => 'Prices rising.'],
]],
]),
]);
// context request fails, falls back to basic
Http::fake([
'https://api.anthropic.com/*' => Http::sequence()
->push([], 500)
->push([
'stop_reason' => 'tool_use',
'content' => [[
'type' => 'tool_use',
'name' => 'submit_prediction',
'input' => ['direction' => 'rising', 'confidence' => 72, 'reasoning' => 'Prices rising.'],
]],
]),
]);
$this->provider->predict(fakePrices(14));
Http::assertSent(function ($request) {
$tools = $request->data()['tools'] ?? [];
return collect($tools)->contains(fn ($t) => $t['name'] === 'submit_prediction');
});
});
it('returns a prediction with Llm source from basic tool use', function (): void {
Http::fake([
'https://api.anthropic.com/*' => Http::sequence()
->push([], 500) // context fails
->push([
'stop_reason' => 'tool_use',
'content' => [[
'type' => 'tool_use',
'name' => 'submit_prediction',
'input' => ['direction' => 'rising', 'confidence' => 72, 'reasoning' => 'Consistent upward trend.'],
]],
]),
]);
$prediction = $this->provider->predict(fakePrices(14));
expect($prediction->direction)->toBe(TrendDirection::Rising)
->and($prediction->source)->toBe(PredictionSource::Llm)
->and($prediction->confidence)->toBe(72)
->and($prediction->reasoning)->toBe('Consistent upward trend.');
});
it('caps confidence at 85', function (): void {
Http::fake([
'https://api.anthropic.com/*' => Http::sequence()
->push([], 500)
->push([
'stop_reason' => 'tool_use',
'content' => [[
'type' => 'tool_use',
'name' => 'submit_prediction',
'input' => ['direction' => 'falling', 'confidence' => 99, 'reasoning' => 'Very confident.'],
]],
]),
]);
$prediction = $this->provider->predict(fakePrices(14));
expect($prediction->confidence)->toBe(85);
});
it('returns null when tool_use block is missing from response', function (): void {
Http::fake([
'https://api.anthropic.com/*' => Http::sequence()
->push([], 500)
->push([
'stop_reason' => 'end_turn',
'content' => [['type' => 'text', 'text' => 'Sorry, I cannot help.']],
]),
]);
expect($this->provider->predict(fakePrices(14)))->toBeNull();
});
it('sends web_search tool during context prediction phase', function (): void {
Http::fake([
'https://api.anthropic.com/*' => Http::sequence()
->push([
'stop_reason' => 'end_turn',
'content' => [['type' => 'text', 'text' => 'Searched and analysed.']],
])
->push([
'stop_reason' => 'tool_use',
'content' => [[
'type' => 'tool_use',
'name' => 'submit_prediction',
'input' => ['direction' => 'flat', 'confidence' => 50, 'reasoning' => 'No clear trend.'],
]],
]),
]);
$this->provider->predict(fakePrices(20));
Http::assertSent(function ($request) {
$tools = $request->data()['tools'] ?? [];
return collect($tools)->contains(fn ($t) => ($t['type'] ?? '') === 'web_search_20250305');
});
});
it('returns LlmWithContext source when context prediction succeeds', function (): void {
Http::fake([
'https://api.anthropic.com/*' => Http::sequence()
->push([
'stop_reason' => 'end_turn',
'content' => [['type' => 'text', 'text' => 'Analysed news.']],
])
->push([
'stop_reason' => 'tool_use',
'content' => [[
'type' => 'tool_use',
'name' => 'submit_prediction',
'input' => ['direction' => 'rising', 'confidence' => 70, 'reasoning' => 'OPEC+ cuts support prices.'],
]],
]),
]);
$prediction = $this->provider->predict(fakePrices(20));
expect($prediction->source)->toBe(PredictionSource::LlmWithContext)
->and($prediction->direction)->toBe(TrendDirection::Rising);
});
it('continues on pause_turn during web search phase', function (): void {
Http::fake([
'https://api.anthropic.com/*' => Http::sequence()
->push([
'stop_reason' => 'pause_turn',
'content' => [['type' => 'server_tool_use', 'name' => 'web_search', 'input' => ['query' => 'Brent crude']]],
])
->push([
'stop_reason' => 'end_turn',
'content' => [['type' => 'text', 'text' => 'Done searching.']],
])
->push([
'stop_reason' => 'tool_use',
'content' => [[
'type' => 'tool_use',
'name' => 'submit_prediction',
'input' => ['direction' => 'falling', 'confidence' => 60, 'reasoning' => 'Demand fears.'],
]],
]),
]);
$prediction = $this->provider->predict(fakePrices(20));
expect($prediction)->not->toBeNull()
->and($prediction->direction)->toBe(TrendDirection::Falling);
Http::assertSentCount(3);
});
it('falls back to basic prediction when context phase fails', function (): void {
Http::fake([
'https://api.anthropic.com/*' => Http::sequence()
->push([], 500) // context search fails
->push([
'stop_reason' => 'tool_use',
'content' => [[
'type' => 'tool_use',
'name' => 'submit_prediction',
'input' => ['direction' => 'rising', 'confidence' => 65, 'reasoning' => 'Rising trend.'],
]],
]),
]);
$prediction = $this->provider->predict(fakePrices(14));
expect($prediction->source)->toBe(PredictionSource::Llm);
});
// --- helpers ---
function fakePrices(int $count): Collection
{
return collect(range(1, $count))->map(fn (int $i) => new BrentPrice([
'date' => now()->subDays($count - $i)->toDateString(),
'price_usd' => 75.0 + $i,
]));
}

View File

@@ -10,7 +10,7 @@ use Illuminate\Foundation\Testing\RefreshDatabase;
uses(RefreshDatabase::class);
it('returns no_signal when there is insufficient price history', function () {
$result = app(NationalFuelPredictionService::class)->predict(FuelType::B7Standard);
$result = app(NationalFuelPredictionService::class)->predict();
expect($result['predicted_direction'])->toBe('stable')
->and($result['signals']['trend']['enabled'])->toBeFalse()
@@ -24,13 +24,13 @@ it('detects rising trend from consistently increasing daily averages', function
for ($daysAgo = 6; $daysAgo >= 0; $daysAgo--) {
StationPrice::factory()->create([
'station_id' => $station->node_id,
'fuel_type' => FuelType::B7Standard,
'fuel_type' => FuelType::E10,
'price_pence' => 14000 + ((6 - $daysAgo) * 100),
'price_effective_at' => now()->subDays($daysAgo)->setTime(12, 0),
]);
}
$result = app(NationalFuelPredictionService::class)->predict(FuelType::B7Standard);
$result = app(NationalFuelPredictionService::class)->predict();
expect($result['signals']['trend']['direction'])->toBe('up')
->and($result['signals']['trend']['enabled'])->toBeTrue()
@@ -44,13 +44,13 @@ it('detects falling trend from consistently decreasing daily averages', function
for ($daysAgo = 6; $daysAgo >= 0; $daysAgo--) {
StationPrice::factory()->create([
'station_id' => $station->node_id,
'fuel_type' => FuelType::B7Standard,
'fuel_type' => FuelType::E10,
'price_pence' => 16000 - ((6 - $daysAgo) * 100),
'price_effective_at' => now()->subDays($daysAgo)->setTime(12, 0),
]);
}
$result = app(NationalFuelPredictionService::class)->predict(FuelType::B7Standard);
$result = app(NationalFuelPredictionService::class)->predict();
expect($result['signals']['trend']['direction'])->toBe('down')
->and($result['predicted_direction'])->toBe('down')
@@ -61,29 +61,70 @@ it('returns current_avg from station_prices_current', function () {
$station = Station::factory()->create();
StationPriceCurrent::factory()->create([
'station_id' => $station->node_id,
'fuel_type' => FuelType::B7Standard,
'fuel_type' => FuelType::E10,
'price_pence' => 14750,
]);
$result = app(NationalFuelPredictionService::class)->predict(FuelType::B7Standard);
$result = app(NationalFuelPredictionService::class)->predict();
expect($result['current_avg'])->toBe(147.5);
});
it('includes all required keys in response', function () {
$result = app(NationalFuelPredictionService::class)->predict(FuelType::B7Standard);
$result = app(NationalFuelPredictionService::class)->predict();
expect($result)->toHaveKeys([
'fuel_type', 'current_avg', 'predicted_direction', 'predicted_change_pence',
'confidence_score', 'confidence_label', 'action', 'reasoning',
'prediction_horizon_days', 'region_key', 'methodology',
'signals',
]);
expect($result)
->toHaveKeys([
'fuel_type', 'current_avg', 'predicted_direction', 'predicted_change_pence',
'confidence_score', 'confidence_label', 'action', 'reasoning',
'prediction_horizon_days', 'region_key', 'methodology',
'signals',
])
->and($result['signals'])->toHaveKeys([
'trend', 'day_of_week', 'brand_behaviour',
'national_momentum', 'regional_momentum', 'price_stickiness',
]);
});
expect($result['signals'])->toHaveKeys([
'trend', 'day_of_week', 'brand_behaviour',
'national_momentum', 'regional_momentum', 'price_stickiness',
]);
it('always returns e10 as fuel_type', function () {
$result = app(NationalFuelPredictionService::class)->predict();
expect($result['fuel_type'])->toBe('e10');
});
it('returns national region_key without coordinates', function () {
$result = app(NationalFuelPredictionService::class)->predict();
expect($result['region_key'])->toBe('national');
});
it('returns regional region_key when coordinates are provided', function () {
$result = app(NationalFuelPredictionService::class)->predict(51.5074, -0.1278);
expect($result['region_key'])->toBe('regional');
});
it('enables regional_momentum signal when coordinates are provided', function () {
$station = Station::factory()->create(['lat' => 51.5, 'lng' => -0.1]);
for ($daysAgo = 6; $daysAgo >= 0; $daysAgo--) {
StationPrice::factory()->create([
'station_id' => $station->node_id,
'fuel_type' => FuelType::E10,
'price_pence' => 14000 + ((6 - $daysAgo) * 100),
'price_effective_at' => now()->subDays($daysAgo)->setTime(12, 0),
]);
}
$result = app(NationalFuelPredictionService::class)->predict(51.5074, -0.1278);
expect($result['signals']['regional_momentum']['enabled'])->toBeTrue();
});
it('disables regional_momentum signal without coordinates', function () {
$result = app(NationalFuelPredictionService::class)->predict();
expect($result['signals']['regional_momentum']['enabled'])->toBeFalse();
});
it('disables trend signal when r_squared is below 0.5', function () {
@@ -94,13 +135,13 @@ it('disables trend signal when r_squared is below 0.5', function () {
foreach ($prices as $daysAgo => $price) {
StationPrice::factory()->create([
'station_id' => $station->node_id,
'fuel_type' => FuelType::B7Standard,
'fuel_type' => FuelType::E10,
'price_pence' => $price,
'price_effective_at' => now()->subDays(count($prices) - 1 - $daysAgo)->setTime(12, 0),
]);
}
$result = app(NationalFuelPredictionService::class)->predict(FuelType::B7Standard);
$result = app(NationalFuelPredictionService::class)->predict();
// Trend signal may be disabled if both 5-day and 14-day lookbacks fail R² threshold
expect($result['signals']['trend']['data_points'])->toBeInt();

View File

@@ -5,6 +5,7 @@ use App\Enums\TrendDirection;
use App\Models\BrentPrice;
use App\Models\PricePrediction;
use App\Services\ApiLogger;
use App\Services\LlmPrediction\OilPredictionProvider;
use App\Services\OilPriceService;
use Illuminate\Foundation\Testing\RefreshDatabase;
use Illuminate\Support\Facades\Http;
@@ -13,7 +14,8 @@ uses(RefreshDatabase::class);
beforeEach(function (): void {
Http::preventStrayRequests();
$this->service = new OilPriceService(new ApiLogger);
$this->provider = Mockery::mock(OilPredictionProvider::class);
$this->service = new OilPriceService(new ApiLogger, $this->provider);
});
// --- fetchBrentPrices ---
@@ -115,173 +117,9 @@ it('returns null when fewer than 14 prices are available for EWMA', function ():
expect($this->service->generateEwmaPrediction($prices))->toBeNull();
});
// --- generateLlmPrediction ---
it('generates an LLM prediction and stores it', function (): void {
$prices = collect(range(1, 14))->map(fn (int $i) => new BrentPrice([
'date' => now()->subDays(14 - $i)->toDateString(),
'price_usd' => 75.0 + $i,
]));
Http::fake([
'https://api.anthropic.com/*' => Http::response([
'content' => [
['text' => '{"direction":"rising","confidence":72,"reasoning":"Consistent upward trend over 14 days."}'],
],
]),
]);
$prediction = $this->service->generateLlmPrediction($prices);
expect($prediction->direction)->toBe(TrendDirection::Rising)
->and($prediction->source)->toBe(PredictionSource::Llm)
->and($prediction->confidence)->toBe(72)
->and($prediction->reasoning)->toBe('Consistent upward trend over 14 days.');
});
it('caps LLM confidence at 85', function (): void {
$prices = collect(range(1, 14))->map(fn (int $i) => new BrentPrice([
'date' => now()->subDays(14 - $i)->toDateString(),
'price_usd' => 75.0,
]));
Http::fake([
'https://api.anthropic.com/*' => Http::response([
'content' => [
['text' => '{"direction":"falling","confidence":99,"reasoning":"Very confident."}'],
],
]),
]);
$prediction = $this->service->generateLlmPrediction($prices);
expect($prediction->confidence)->toBe(85);
});
it('returns null when LLM returns malformed JSON', function (): void {
$prices = collect(range(1, 14))->map(fn (int $i) => new BrentPrice([
'date' => now()->subDays(14 - $i)->toDateString(),
'price_usd' => 75.0,
]));
Http::fake([
'https://api.anthropic.com/*' => Http::response([
'content' => [['text' => 'Sorry, I cannot help with that.']],
]),
]);
expect($this->service->generateLlmPrediction($prices))->toBeNull();
});
// --- generateLlmPredictionWithContext ---
it('generates LLM prediction with context and returns LlmWithContext source', function (): void {
config(['services.anthropic.api_key' => 'test-key']);
$prices = collect(range(1, 20))->map(fn (int $i) => new BrentPrice([
'date' => now()->subDays(20 - $i)->toDateString(),
'price_usd' => 80.0 + $i * 0.5,
]));
Http::fake([
'https://api.anthropic.com/*' => Http::response([
'content' => [['type' => 'text', 'text' => '{"direction":"rising","confidence":72,"reasoning":"OPEC+ extended cuts while prices trend upward."}']],
'stop_reason' => 'end_turn',
]),
]);
$prediction = $this->service->generateLlmPredictionWithContext($prices);
expect($prediction)->not->toBeNull()
->and($prediction->direction)->toBe(TrendDirection::Rising)
->and($prediction->confidence)->toBe(72)
->and($prediction->source)->toBe(PredictionSource::LlmWithContext)
->and($prediction->reasoning)->toBe('OPEC+ extended cuts while prices trend upward.');
Http::assertSentCount(1);
});
it('sends web_search tool in the context prediction request', function (): void {
config(['services.anthropic.api_key' => 'test-key']);
$prices = collect(range(1, 20))->map(fn (int $i) => new BrentPrice([
'date' => now()->subDays(20 - $i)->toDateString(),
'price_usd' => 80.0,
]));
Http::fake([
'https://api.anthropic.com/*' => Http::response([
'content' => [['type' => 'text', 'text' => '{"direction":"flat","confidence":50,"reasoning":"No clear trend."}']],
'stop_reason' => 'end_turn',
]),
]);
$this->service->generateLlmPredictionWithContext($prices);
Http::assertSent(function ($request) {
$tools = $request->data()['tools'] ?? [];
return collect($tools)->contains(fn ($t) => $t['type'] === 'web_search_20250305');
});
});
it('does not include EWMA indicators in the context prediction prompt', function (): void {
config(['services.anthropic.api_key' => 'test-key']);
$prices = collect(range(1, 20))->map(fn (int $i) => new BrentPrice([
'date' => now()->subDays(20 - $i)->toDateString(),
'price_usd' => 80.0,
]));
Http::fake([
'https://api.anthropic.com/*' => Http::response([
'content' => [['type' => 'text', 'text' => '{"direction":"flat","confidence":50,"reasoning":"No clear trend."}']],
'stop_reason' => 'end_turn',
]),
]);
$this->service->generateLlmPredictionWithContext($prices);
Http::assertSent(function ($request) {
$content = $request->data()['messages'][0]['content'] ?? '';
return ! str_contains($content, 'EWMA') && ! str_contains($content, 'Pre-computed');
});
});
it('continues on pause_turn and returns final answer', function (): void {
config(['services.anthropic.api_key' => 'test-key']);
$prices = collect(range(1, 20))->map(fn (int $i) => new BrentPrice([
'date' => now()->subDays(20 - $i)->toDateString(),
'price_usd' => 80.0,
]));
Http::fake([
'https://api.anthropic.com/*' => Http::sequence()
->push([
'content' => [['type' => 'server_tool_use', 'id' => 'sttool_1', 'name' => 'web_search', 'input' => ['query' => 'Brent crude news']]],
'stop_reason' => 'pause_turn',
])
->push([
'content' => [['type' => 'text', 'text' => '{"direction":"falling","confidence":60,"reasoning":"Demand fears weigh on prices."}']],
'stop_reason' => 'end_turn',
]),
]);
$prediction = $this->service->generateLlmPredictionWithContext($prices);
expect($prediction)->not->toBeNull()
->and($prediction->direction)->toBe(TrendDirection::Falling);
Http::assertSentCount(2);
});
// --- generatePrediction (orchestrator) ---
it('uses LLM with context when API key is configured', function (): void {
config(['services.anthropic.api_key' => 'test-key']);
it('stores both EWMA and LLM predictions when provider succeeds', function (): void {
BrentPrice::insert(
collect(range(1, 20))->map(fn (int $i) => [
'date' => now()->subDays(20 - $i)->toDateString(),
@@ -289,12 +127,14 @@ it('uses LLM with context when API key is configured', function (): void {
])->all()
);
Http::fake([
'https://api.anthropic.com/*' => Http::response([
'content' => [['type' => 'text', 'text' => '{"direction":"rising","confidence":70,"reasoning":"Trend is up."}']],
'stop_reason' => 'end_turn',
]),
]);
$this->provider->shouldReceive('predict')->once()->andReturn(new PricePrediction([
'predicted_for' => now()->toDateString(),
'source' => PredictionSource::LlmWithContext,
'direction' => TrendDirection::Rising,
'confidence' => 70,
'reasoning' => 'Trend is up.',
'generated_at' => now(),
]));
$prediction = $this->service->generatePrediction();
@@ -302,33 +142,31 @@ it('uses LLM with context when API key is configured', function (): void {
->and(PricePrediction::count())->toBe(2);
});
it('falls back to plain LLM when context method fails', function (): void {
config(['services.anthropic.api_key' => 'test-key']);
it('returns LLM prediction when provider succeeds', function (): void {
BrentPrice::insert(
collect(range(1, 20))->map(fn (int $i) => [
'date' => now()->subDays(20 - $i)->toDateString(),
'price_usd' => 75.0 + ($i * 0.8),
'price_usd' => 75.0 + $i,
])->all()
);
Http::fake([
'https://api.anthropic.com/*' => Http::sequence()
->push([], 500)
->push([
'content' => [['text' => '{"direction":"rising","confidence":70,"reasoning":"Trend up."}']],
]),
$llmPrediction = new PricePrediction([
'predicted_for' => now()->toDateString(),
'source' => PredictionSource::Llm,
'direction' => TrendDirection::Rising,
'confidence' => 65,
'reasoning' => 'Rising trend.',
'generated_at' => now(),
]);
$this->provider->shouldReceive('predict')->once()->andReturn($llmPrediction);
$prediction = $this->service->generatePrediction();
expect($prediction->source)->toBe(PredictionSource::Llm)
->and(PricePrediction::count())->toBe(2);
expect($prediction->source)->toBe(PredictionSource::Llm);
});
it('falls back to EWMA when both LLM methods fail', function (): void {
config(['services.anthropic.api_key' => 'test-key']);
it('falls back to EWMA when provider returns null', function (): void {
BrentPrice::insert(
collect(range(1, 20))->map(fn (int $i) => [
'date' => now()->subDays(20 - $i)->toDateString(),
@@ -336,9 +174,7 @@ it('falls back to EWMA when both LLM methods fail', function (): void {
])->all()
);
Http::fake([
'https://api.anthropic.com/*' => Http::response([], 500),
]);
$this->provider->shouldReceive('predict')->once()->andReturn(null);
$prediction = $this->service->generatePrediction();
@@ -352,6 +188,8 @@ it('returns null when there is insufficient price data', function (): void {
['date' => now()->subDay()->toDateString(), 'price_usd' => 76.0],
]);
$this->provider->shouldNotReceive('predict');
expect($this->service->generatePrediction())->toBeNull()
->and(PricePrediction::count())->toBe(0);
});