diff --git a/app/Console/Commands/ImportPostcodes.php b/app/Console/Commands/ImportPostcodes.php new file mode 100644 index 0000000..cfa6c7d --- /dev/null +++ b/app/Console/Commands/ImportPostcodes.php @@ -0,0 +1,122 @@ +option('file'); + + if ($file === null || ! is_readable($file)) { + $this->error('--file is required and must be a readable path to an ONSPD CSV.'); + + return self::FAILURE; + } + + $handle = fopen($file, 'r'); + + if ($handle === false) { + $this->error("Unable to open {$file}."); + + return self::FAILURE; + } + + $header = fgetcsv($handle); + + if ($header === false) { + $this->error('CSV is empty.'); + fclose($handle); + + return self::FAILURE; + } + + $headerCounts = array_count_values(array_map('strtolower', $header)); + + foreach (['pcd', 'lat', 'long'] as $required) { + if (($headerCounts[$required] ?? 0) > 1) { + $this->error("Column '{$required}' appears more than once — refusing to import."); + fclose($handle); + + return self::FAILURE; + } + } + + $columns = array_change_key_case(array_flip($header), CASE_LOWER); + + foreach (['pcd', 'lat', 'long'] as $required) { + if (! isset($columns[$required])) { + $this->error("Missing required column '{$required}'."); + fclose($handle); + + return self::FAILURE; + } + } + + $hasDoterm = isset($columns['doterm']); + + DB::table('postcodes')->truncate(); + DB::table('outcodes')->truncate(); + + $buffer = []; + $imported = 0; + + while (($row = fgetcsv($handle)) !== false) { + if ($hasDoterm && trim((string) ($row[$columns['doterm']] ?? '')) !== '') { + continue; + } + + $lat = trim((string) ($row[$columns['lat']] ?? '')); + $lng = trim((string) ($row[$columns['long']] ?? '')); + + if ($lat === '' || $lng === '') { + continue; + } + + $pcd = strtoupper(preg_replace('/\s+/', '', (string) $row[$columns['pcd']])); + + if ($pcd === '' || strlen($pcd) < 5) { + continue; + } + + $buffer[] = [ + 'postcode' => $pcd, + 'outcode' => substr($pcd, 0, strlen($pcd) - 3), + 'lat' => (float) $lat, + 'lng' => (float) $lng, + ]; + + if (count($buffer) >= self::CHUNK_SIZE) { + DB::table('postcodes')->insert($buffer); + $imported += count($buffer); + $buffer = []; + } + } + + if ($buffer !== []) { + DB::table('postcodes')->insert($buffer); + $imported += count($buffer); + } + + fclose($handle); + + DB::statement( + 'INSERT INTO outcodes (outcode, lat, lng) + SELECT outcode, AVG(lat), AVG(lng) FROM postcodes GROUP BY outcode' + ); + + $this->info("Imported {$imported} postcodes."); + $this->info('Derived '.DB::table('outcodes')->count().' outcode centroids.'); + + return self::SUCCESS; + } +} diff --git a/app/Models/Outcode.php b/app/Models/Outcode.php new file mode 100644 index 0000000..472debc --- /dev/null +++ b/app/Models/Outcode.php @@ -0,0 +1,26 @@ + 'float', + 'lng' => 'float', + ]; + } +} diff --git a/app/Models/Postcode.php b/app/Models/Postcode.php new file mode 100644 index 0000000..dfd1084 --- /dev/null +++ b/app/Models/Postcode.php @@ -0,0 +1,26 @@ + 'float', + 'lng' => 'float', + ]; + } +} diff --git a/app/Services/PostcodeService.php b/app/Services/PostcodeService.php index e05af9d..8cc1c93 100644 --- a/app/Services/PostcodeService.php +++ b/app/Services/PostcodeService.php @@ -2,6 +2,8 @@ namespace App\Services; +use App\Models\Outcode; +use App\Models\Postcode; use Illuminate\Support\Facades\Cache; use Illuminate\Support\Facades\Http; use Illuminate\Support\Facades\Log; @@ -24,7 +26,16 @@ class PostcodeService public function resolve(string $query): ?LocationResult { $query = trim($query); - $cacheKey = 'postcode:'.strtolower(preg_replace('/\s+/', '', $query)); + + if ($this->isFullPostcode($query)) { + return $this->lookupLocalPostcode($query) ?? $this->lookupPostcode($query); + } + + if ($this->isOutcode($query)) { + return $this->lookupLocalOutcode($query) ?? $this->lookupOutcode($query); + } + + $cacheKey = 'place:'.strtolower(preg_replace('/\s+/', '', $query)); $cached = Cache::get($cacheKey); @@ -32,11 +43,7 @@ class PostcodeService return $cached; } - $result = match (true) { - $this->isFullPostcode($query) => $this->lookupPostcode($query), - $this->isOutcode($query) => $this->lookupOutcode($query), - default => $this->lookupPlace($query), - }; + $result = $this->lookupPlace($query); if ($result !== null) { Cache::put($cacheKey, $result, self::CACHE_TTL); @@ -45,6 +52,11 @@ class PostcodeService return $result; } + private function normalisePostcode(string $value): string + { + return strtoupper(preg_replace('/\s+/', '', $value)); + } + private function isFullPostcode(string $query): bool { return (bool) preg_match('/^[A-Z]{1,2}[0-9][0-9A-Z]?\s*[0-9][A-Z]{2}$/i', $query); @@ -55,9 +67,55 @@ class PostcodeService return (bool) preg_match('/^[A-Z]{1,2}[0-9][0-9A-Z]?$/i', $query); } + private function lookupLocalPostcode(string $postcode): ?LocationResult + { + $normalised = $this->normalisePostcode($postcode); + + $row = Postcode::find($normalised); + + if ($row === null) { + return null; + } + + return new LocationResult( + query: $postcode, + displayName: $this->formatPostcode($normalised), + lat: $row->lat, + lng: $row->lng, + ); + } + + private function lookupLocalOutcode(string $outcode): ?LocationResult + { + $normalised = strtoupper(trim($outcode)); + + $row = Outcode::find($normalised); + + if ($row === null) { + return null; + } + + return new LocationResult( + query: $outcode, + displayName: $normalised, + lat: $row->lat, + lng: $row->lng, + ); + } + + private function formatPostcode(string $normalised): string + { + // Insert the single space before the last 3 chars ("SW1A1AA" -> "SW1A 1AA"). + if (strlen($normalised) < 5) { + return $normalised; + } + + return substr($normalised, 0, -3).' '.substr($normalised, -3); + } + private function lookupPostcode(string $postcode): ?LocationResult { - $normalised = strtoupper(preg_replace('/\s+/', '', $postcode)); + $normalised = $this->normalisePostcode($postcode); $url = self::BASE_URL.'/postcodes/'.$normalised; try { @@ -69,12 +127,34 @@ class PostcodeService $data = $response->json('result'); - return new LocationResult( + if (! is_array($data) || ! isset($data['postcode'], $data['latitude'], $data['longitude'])) { + return null; + } + + $result = new LocationResult( query: $postcode, displayName: $data['postcode'], lat: $data['latitude'], lng: $data['longitude'], ); + + try { + Postcode::updateOrCreate( + ['postcode' => $normalised], + [ + 'outcode' => substr($normalised, 0, strlen($normalised) - 3), + 'lat' => $data['latitude'], + 'lng' => $data['longitude'], + ], + ); + } catch (Throwable $e) { + Log::warning('PostcodeService: failed to persist postcode after HTTP fallback', [ + 'postcode' => $normalised, + 'error' => $e->getMessage(), + ]); + } + + return $result; } catch (Throwable $e) { Log::error('PostcodeService: postcode lookup failed', [ 'postcode' => $postcode, @@ -99,12 +179,33 @@ class PostcodeService $data = $response->json('result'); - return new LocationResult( + if (! is_array($data) || ! isset($data['outcode'], $data['latitude'], $data['longitude'])) { + return null; + } + + $result = new LocationResult( query: $outcode, displayName: $data['outcode'], lat: $data['latitude'], lng: $data['longitude'], ); + + try { + Outcode::updateOrCreate( + ['outcode' => $normalised], + [ + 'lat' => $data['latitude'], + 'lng' => $data['longitude'], + ], + ); + } catch (Throwable $e) { + Log::warning('PostcodeService: failed to persist outcode after HTTP fallback', [ + 'outcode' => $normalised, + 'error' => $e->getMessage(), + ]); + } + + return $result; } catch (Throwable $e) { Log::error('PostcodeService: outcode lookup failed', [ 'outcode' => $outcode, diff --git a/database/migrations/2026_04_22_100000_create_postcodes_table.php b/database/migrations/2026_04_22_100000_create_postcodes_table.php new file mode 100644 index 0000000..cd50ed0 --- /dev/null +++ b/database/migrations/2026_04_22_100000_create_postcodes_table.php @@ -0,0 +1,23 @@ +string('postcode', 7)->primary()->comment('Normalised: uppercase, no spaces'); + $table->string('outcode', 4)->index(); + $table->decimal('lat', 10, 7); + $table->decimal('lng', 10, 7); + }); + } + + public function down(): void + { + Schema::dropIfExists('postcodes'); + } +}; diff --git a/database/migrations/2026_04_22_100001_create_outcodes_table.php b/database/migrations/2026_04_22_100001_create_outcodes_table.php new file mode 100644 index 0000000..024776f --- /dev/null +++ b/database/migrations/2026_04_22_100001_create_outcodes_table.php @@ -0,0 +1,22 @@ +string('outcode', 4)->primary(); + $table->decimal('lat', 10, 7); + $table->decimal('lng', 10, 7); + }); + } + + public function down(): void + { + Schema::dropIfExists('outcodes'); + } +}; diff --git a/docs/superpowers/plans/2026-04-22-self-hosted-postcodes.md b/docs/superpowers/plans/2026-04-22-self-hosted-postcodes.md new file mode 100644 index 0000000..e3b1ffa --- /dev/null +++ b/docs/superpowers/plans/2026-04-22-self-hosted-postcodes.md @@ -0,0 +1,1071 @@ +# UK Postcode Self-Hosting Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Resolve full UK postcodes and outcodes to lat/lng from a local MySQL table sourced from the ONS Postcode Directory (ONSPD), keeping postcodes.io only as a fallback for place names and unknown postcodes. + +**Architecture:** Two new tables (`postcodes`, `outcodes`) seeded from the quarterly ONSPD CSV by an idempotent Artisan command (`postcodes:import --file=...`). `PostcodeService::resolve()` checks local DB first; when a full postcode or outcode lookup misses, it falls through to the existing HTTP client and persists the successful result back into the table. Place-name lookups still go to postcodes.io (not in ONSPD). The 30-day `Cache::put` wrapper is removed for postcode/outcode paths (DB IS the cache) and re-scoped to a `place:` prefix for place names only. + +**Tech Stack:** Laravel 13, MySQL, Pest 4, PHP 8.4. + +--- + +## File Structure + +**Create:** +- `database/migrations/2026_04_22_100000_create_postcodes_table.php` +- `database/migrations/2026_04_22_100001_create_outcodes_table.php` +- `app/Models/Postcode.php` +- `app/Models/Outcode.php` +- `app/Console/Commands/ImportPostcodes.php` +- `tests/Feature/Console/ImportPostcodesTest.php` + +**Modify:** +- `app/Services/PostcodeService.php` — add local lookup + HTTP fallback persistence +- `tests/Unit/Services/PostcodeServiceTest.php` — update existing HTTP-heavy tests for new DB-first flow +- `resources/js/views/Home.vue` — add ONS/Royal Mail/OS attribution line in footer + +--- + +### Task 1: Create `postcodes` and `outcodes` tables + +**Files:** +- Create: `database/migrations/2026_04_22_100000_create_postcodes_table.php` +- Create: `database/migrations/2026_04_22_100001_create_outcodes_table.php` + +- [ ] **Step 1: Generate migrations** + +Run: +```bash +php artisan make:migration create_postcodes_table --no-interaction +php artisan make:migration create_outcodes_table --no-interaction +``` + +Rename the two generated files so their timestamps are `2026_04_22_100000` and `2026_04_22_100001` respectively (so they sit together in order). + +- [ ] **Step 2: Fill in `create_postcodes_table`** + +Replace file contents with: + +```php +string('postcode', 7)->primary()->comment('Normalised: uppercase, no spaces'); + $table->string('outcode', 4)->index(); + $table->decimal('lat', 10, 7); + $table->decimal('lng', 10, 7); + }); + } + + public function down(): void + { + Schema::dropIfExists('postcodes'); + } +}; +``` + +- [ ] **Step 3: Fill in `create_outcodes_table`** + +Replace file contents with: + +```php +string('outcode', 4)->primary(); + $table->decimal('lat', 10, 7); + $table->decimal('lng', 10, 7); + }); + } + + public function down(): void + { + Schema::dropIfExists('outcodes'); + } +}; +``` + +- [ ] **Step 4: Run migrations** + +Run: `php artisan migrate` +Expected: both tables created, no errors. + +- [ ] **Step 5: Commit** + +```bash +git add database/migrations/2026_04_22_100000_create_postcodes_table.php \ + database/migrations/2026_04_22_100001_create_outcodes_table.php +git commit -m "feat: add postcodes and outcodes tables for self-hosted lookup" +``` + +--- + +### Task 2: Create `Postcode` and `Outcode` Eloquent models + +**Files:** +- Create: `app/Models/Postcode.php` +- Create: `app/Models/Outcode.php` + +- [ ] **Step 1: Generate both models** + +Run: +```bash +php artisan make:model Postcode --no-interaction +php artisan make:model Outcode --no-interaction +``` + +- [ ] **Step 2: Configure `Postcode` model** + +Replace `app/Models/Postcode.php` with: + +```php + 'float', + 'lng' => 'float', + ]; + } +} +``` + +- [ ] **Step 3: Configure `Outcode` model** + +Replace `app/Models/Outcode.php` with: + +```php + 'float', + 'lng' => 'float', + ]; + } +} +``` + +- [ ] **Step 4: Commit** + +```bash +git add app/Models/Postcode.php app/Models/Outcode.php +git commit -m "feat: add Postcode and Outcode Eloquent models" +``` + +--- + +### Task 3: Resolve full postcodes from local DB first + +**Files:** +- Modify: `app/Services/PostcodeService.php` +- Modify: `tests/Unit/Services/PostcodeServiceTest.php` + +- [ ] **Step 1: Add failing test — local DB hit skips HTTP** + +Append to `tests/Unit/Services/PostcodeServiceTest.php`: + +```php +// --- Local DB (full postcode) --- + +it('resolves a full postcode from local DB without calling HTTP', function (): void { + \App\Models\Postcode::create([ + 'postcode' => 'SW1A1AA', + 'outcode' => 'SW1A', + 'lat' => 51.501009, + 'lng' => -0.141588, + ]); + + Http::fake(); // any HTTP call will be recorded + + $result = $this->service->resolve('SW1A 1AA'); + + expect($result)->toBeInstanceOf(LocationResult::class) + ->and($result->displayName)->toBe('SW1A 1AA') + ->and($result->lat)->toBe(51.501009) + ->and($result->lng)->toBe(-0.141588); + + Http::assertNothingSent(); +}); +``` + +- [ ] **Step 2: Run test to confirm it fails** + +Run: `timeout 10 php artisan test --compact --filter='resolves a full postcode from local DB'` +Expected: FAIL — test passes the resolve call but currently no local lookup exists, so HTTP `Http::fake()` returns empty and `resolve()` returns null (or sends HTTP and fails the `assertNothingSent`). + +- [ ] **Step 3: Implement local full-postcode lookup** + +In `app/Services/PostcodeService.php`, add a new private method above `lookupPostcode`: + +```php + private function lookupLocalPostcode(string $postcode): ?LocationResult + { + $normalised = strtoupper(preg_replace('/\s+/', '', $postcode)); + + $row = \App\Models\Postcode::find($normalised); + + if ($row === null) { + return null; + } + + return new LocationResult( + query: $postcode, + displayName: $this->formatPostcode($normalised), + lat: $row->lat, + lng: $row->lng, + ); + } + + private function formatPostcode(string $normalised): string + { + // Insert the single space before the last 3 chars ("SW1A1AA" -> "SW1A 1AA"). + if (strlen($normalised) < 5) { + return $normalised; + } + + return substr($normalised, 0, -3).' '.substr($normalised, -3); + } +``` + +Then modify the `resolve()` method — change the full-postcode branch so it checks local first: + +Replace this existing block inside `resolve()`: + +```php + $result = match (true) { + $this->isFullPostcode($query) => $this->lookupPostcode($query), + $this->isOutcode($query) => $this->lookupOutcode($query), + default => $this->lookupPlace($query), + }; +``` + +With: + +```php + $result = match (true) { + $this->isFullPostcode($query) => $this->lookupLocalPostcode($query) ?? $this->lookupPostcode($query), + $this->isOutcode($query) => $this->lookupOutcode($query), + default => $this->lookupPlace($query), + }; +``` + +- [ ] **Step 4: Run test — expect PASS** + +Run: `timeout 10 php artisan test --compact --filter='resolves a full postcode from local DB'` +Expected: PASS. + +- [ ] **Step 5: Run full PostcodeServiceTest — existing HTTP-based tests must still pass (local table is empty for those)** + +Run: `timeout 10 php artisan test --compact --filter=PostcodeServiceTest` +Expected: all tests pass. The existing HTTP tests start with an empty DB, so they fall through to the HTTP fake as before. + +- [ ] **Step 6: Commit** + +```bash +git add app/Services/PostcodeService.php tests/Unit/Services/PostcodeServiceTest.php +git commit -m "feat: resolve full postcodes from local DB before HTTP" +``` + +--- + +### Task 4: Resolve outcodes from local DB first + +**Files:** +- Modify: `app/Services/PostcodeService.php` +- Modify: `tests/Unit/Services/PostcodeServiceTest.php` + +- [ ] **Step 1: Add failing test — local outcode hit skips HTTP** + +Append to `tests/Unit/Services/PostcodeServiceTest.php`: + +```php +// --- Local DB (outcode) --- + +it('resolves an outcode from local DB without calling HTTP', function (): void { + \App\Models\Outcode::create([ + 'outcode' => 'PE7', + 'lat' => 52.536397, + 'lng' => -0.210181, + ]); + + Http::fake(); + + $result = $this->service->resolve('PE7'); + + expect($result)->toBeInstanceOf(LocationResult::class) + ->and($result->displayName)->toBe('PE7') + ->and($result->lat)->toBe(52.536397) + ->and($result->lng)->toBe(-0.210181); + + Http::assertNothingSent(); +}); +``` + +- [ ] **Step 2: Run test to confirm it fails** + +Run: `timeout 10 php artisan test --compact --filter='resolves an outcode from local DB'` +Expected: FAIL. + +- [ ] **Step 3: Implement local outcode lookup** + +In `app/Services/PostcodeService.php`, add after `lookupLocalPostcode`: + +```php + private function lookupLocalOutcode(string $outcode): ?LocationResult + { + $normalised = strtoupper(trim($outcode)); + + $row = \App\Models\Outcode::find($normalised); + + if ($row === null) { + return null; + } + + return new LocationResult( + query: $outcode, + displayName: $normalised, + lat: $row->lat, + lng: $row->lng, + ); + } +``` + +Update the match in `resolve()` again: + +```php + $result = match (true) { + $this->isFullPostcode($query) => $this->lookupLocalPostcode($query) ?? $this->lookupPostcode($query), + $this->isOutcode($query) => $this->lookupLocalOutcode($query) ?? $this->lookupOutcode($query), + default => $this->lookupPlace($query), + }; +``` + +- [ ] **Step 4: Run test — expect PASS** + +Run: `timeout 10 php artisan test --compact --filter='resolves an outcode from local DB'` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add app/Services/PostcodeService.php tests/Unit/Services/PostcodeServiceTest.php +git commit -m "feat: resolve outcodes from local DB before HTTP" +``` + +--- + +### Task 5: Persist HTTP fallback results into local DB + +**Files:** +- Modify: `app/Services/PostcodeService.php` +- Modify: `tests/Unit/Services/PostcodeServiceTest.php` + +- [ ] **Step 1: Add failing test — HTTP fallback persists postcode** + +Append to `tests/Unit/Services/PostcodeServiceTest.php`: + +```php +// --- HTTP fallback persistence --- + +it('persists a full postcode resolved via HTTP fallback', function (): void { + Http::fake([ + '*/postcodes/SW1A1AA' => Http::response([ + 'status' => 200, + 'result' => [ + 'postcode' => 'SW1A 1AA', + 'latitude' => 51.501009, + 'longitude' => -0.141588, + ], + ]), + ]); + + $this->service->resolve('SW1A 1AA'); + + $row = \App\Models\Postcode::find('SW1A1AA'); + + expect($row)->not->toBeNull() + ->and($row->outcode)->toBe('SW1A') + ->and((float) $row->lat)->toBe(51.501009) + ->and((float) $row->lng)->toBe(-0.141588); +}); + +it('persists an outcode resolved via HTTP fallback', function (): void { + Http::fake([ + '*/outcodes/PE7' => Http::response([ + 'status' => 200, + 'result' => [ + 'outcode' => 'PE7', + 'latitude' => 52.536397, + 'longitude' => -0.210181, + ], + ]), + ]); + + $this->service->resolve('PE7'); + + $row = \App\Models\Outcode::find('PE7'); + + expect($row)->not->toBeNull() + ->and((float) $row->lat)->toBe(52.536397) + ->and((float) $row->lng)->toBe(-0.210181); +}); +``` + +- [ ] **Step 2: Run tests — expect FAIL (rows not persisted yet)** + +Run: `timeout 10 php artisan test --compact --filter='persists'` +Expected: both tests FAIL — rows are null. + +- [ ] **Step 3: Add persistence after successful HTTP lookup** + +In `app/Services/PostcodeService.php`, update `lookupPostcode` — after the `LocationResult` is constructed, persist: + +Replace the current body of `lookupPostcode` with: + +```php + private function lookupPostcode(string $postcode): ?LocationResult + { + $normalised = strtoupper(preg_replace('/\s+/', '', $postcode)); + $url = self::BASE_URL.'/postcodes/'.$normalised; + + try { + $response = $this->apiLogger->send('postcodes_io', 'GET', $url, fn () => Http::timeout(10)->get($url)); + + if (! $response->successful()) { + return null; + } + + $data = $response->json('result'); + + $result = new LocationResult( + query: $postcode, + displayName: $data['postcode'], + lat: $data['latitude'], + lng: $data['longitude'], + ); + + \App\Models\Postcode::updateOrCreate( + ['postcode' => $normalised], + [ + 'outcode' => substr($normalised, 0, strlen($normalised) - 3), + 'lat' => $data['latitude'], + 'lng' => $data['longitude'], + ], + ); + + return $result; + } catch (Throwable $e) { + Log::error('PostcodeService: postcode lookup failed', [ + 'postcode' => $postcode, + 'error' => $e->getMessage(), + ]); + + return null; + } + } +``` + +Replace the current body of `lookupOutcode` with: + +```php + private function lookupOutcode(string $outcode): ?LocationResult + { + $normalised = strtoupper(trim($outcode)); + $url = self::BASE_URL.'/outcodes/'.$normalised; + + try { + $response = $this->apiLogger->send('postcodes_io', 'GET', $url, fn () => Http::timeout(10)->get($url)); + + if (! $response->successful()) { + return null; + } + + $data = $response->json('result'); + + $result = new LocationResult( + query: $outcode, + displayName: $data['outcode'], + lat: $data['latitude'], + lng: $data['longitude'], + ); + + \App\Models\Outcode::updateOrCreate( + ['outcode' => $normalised], + [ + 'lat' => $data['latitude'], + 'lng' => $data['longitude'], + ], + ); + + return $result; + } catch (Throwable $e) { + Log::error('PostcodeService: outcode lookup failed', [ + 'outcode' => $outcode, + 'error' => $e->getMessage(), + ]); + + return null; + } + } +``` + +- [ ] **Step 4: Run persistence tests — expect PASS** + +Run: `timeout 10 php artisan test --compact --filter='persists'` +Expected: PASS. + +- [ ] **Step 5: Run full PostcodeServiceTest file** + +Run: `timeout 10 php artisan test --compact --filter=PostcodeServiceTest` +Expected: all tests pass. + +- [ ] **Step 6: Commit** + +```bash +git add app/Services/PostcodeService.php tests/Unit/Services/PostcodeServiceTest.php +git commit -m "feat: persist postcodes.io fallback results into local DB" +``` + +--- + +### Task 6: Remove stale 30-day Cache for postcodes/outcodes; scope cache to places + +**Files:** +- Modify: `app/Services/PostcodeService.php` +- Modify: `tests/Unit/Services/PostcodeServiceTest.php` + +- [ ] **Step 1: Update the existing caching test so it only covers places** + +In `tests/Unit/Services/PostcodeServiceTest.php`, replace the existing test titled `'caches a successful resolution for 30 days'` with: + +```php +it('caches a successful place resolution for 30 days', function (): void { + Http::fake([ + '*/places*' => Http::response([ + 'status' => 200, + 'result' => [[ + 'name_1' => 'Manchester', + 'latitude' => 53.480957, + 'longitude' => -2.237428, + ]], + ]), + ]); + + $this->service->resolve('Manchester'); + $this->service->resolve('Manchester'); + + Http::assertSentCount(1); + expect(Cache::get('place:manchester'))->toBeInstanceOf(LocationResult::class); +}); + +it('does not cache postcode resolutions in the Cache store (DB is the cache)', function (): void { + Http::fake([ + '*/postcodes/SW1A1AA' => Http::response([ + 'status' => 200, + 'result' => [ + 'postcode' => 'SW1A 1AA', + 'latitude' => 51.501009, + 'longitude' => -0.141588, + ], + ]), + ]); + + $this->service->resolve('SW1A 1AA'); + + expect(Cache::get('postcode:sw1a1aa'))->toBeNull() + ->and(\App\Models\Postcode::find('SW1A1AA'))->not->toBeNull(); +}); +``` + +- [ ] **Step 2: Run the two cache tests — expect FAIL (cache key still written under old prefix)** + +Run: `timeout 10 php artisan test --compact --filter='cache'` +Expected: FAIL — old code still writes under `postcode:...` for all lookup types. + +- [ ] **Step 3: Scope the Cache wrapper to place names only** + +In `app/Services/PostcodeService.php`, replace the `resolve()` method with: + +```php + public function resolve(string $query): ?LocationResult + { + $query = trim($query); + + if ($this->isFullPostcode($query)) { + return $this->lookupLocalPostcode($query) ?? $this->lookupPostcode($query); + } + + if ($this->isOutcode($query)) { + return $this->lookupLocalOutcode($query) ?? $this->lookupOutcode($query); + } + + $cacheKey = 'place:'.strtolower(preg_replace('/\s+/', '', $query)); + + $cached = Cache::get($cacheKey); + + if ($cached instanceof LocationResult) { + return $cached; + } + + $result = $this->lookupPlace($query); + + if ($result !== null) { + Cache::put($cacheKey, $result, self::CACHE_TTL); + } + + return $result; + } +``` + +- [ ] **Step 4: Run PostcodeServiceTest — expect PASS** + +Run: `timeout 10 php artisan test --compact --filter=PostcodeServiceTest` +Expected: all tests pass. If an older test still references the `postcode:` cache key for a place lookup, update it to `place:` — nothing else should break. + +- [ ] **Step 5: Commit** + +```bash +git add app/Services/PostcodeService.php tests/Unit/Services/PostcodeServiceTest.php +git commit -m "refactor: scope postcode cache to place names, DB is authoritative for postcodes" +``` + +--- + +### Task 7: `postcodes:import` command — parse ONSPD CSV and load `postcodes` table + +**Files:** +- Create: `app/Console/Commands/ImportPostcodes.php` +- Create: `tests/Feature/Console/ImportPostcodesTest.php` + +- [ ] **Step 1: Generate the command skeleton** + +Run: +```bash +php artisan make:command ImportPostcodes --no-interaction +php artisan make:test --pest Console/ImportPostcodesTest --no-interaction +``` + +- [ ] **Step 2: Write failing test — small fixture CSV populates `postcodes`** + +Replace `tests/Feature/Console/ImportPostcodesTest.php` with: + +```php +artisan('postcodes:import', ['--file' => $path]) + ->assertSuccessful(); + + expect(Postcode::count())->toBe(2) + ->and(Postcode::find('SW1A1AA')->outcode)->toBe('SW1A') + ->and(Postcode::find('M11AD')->outcode)->toBe('M1'); +}); +``` + +- [ ] **Step 3: Run test — expect FAIL** + +Run: `timeout 10 php artisan test --compact --filter='imports active postcodes'` +Expected: FAIL — command not implemented. + +- [ ] **Step 4: Implement the command** + +Replace `app/Console/Commands/ImportPostcodes.php` with: + +```php +option('file'); + + if ($file === null || ! is_readable($file)) { + $this->error('--file is required and must be a readable path to an ONSPD CSV.'); + + return self::FAILURE; + } + + $handle = fopen($file, 'r'); + + if ($handle === false) { + throw new RuntimeException("Unable to open {$file}"); + } + + $header = fgetcsv($handle); + + if ($header === false) { + $this->error('CSV is empty.'); + fclose($handle); + + return self::FAILURE; + } + + $columns = array_change_key_case(array_flip($header), CASE_LOWER); + + foreach (['pcd', 'lat', 'long'] as $required) { + if (! isset($columns[$required])) { + $this->error("Missing required column '{$required}'."); + fclose($handle); + + return self::FAILURE; + } + } + + $hasDoterm = isset($columns['doterm']); + + DB::table('postcodes')->truncate(); + DB::table('outcodes')->truncate(); + + $buffer = []; + $imported = 0; + + while (($row = fgetcsv($handle)) !== false) { + if ($hasDoterm && trim((string) ($row[$columns['doterm']] ?? '')) !== '') { + continue; + } + + $lat = trim((string) ($row[$columns['lat']] ?? '')); + $lng = trim((string) ($row[$columns['long']] ?? '')); + + if ($lat === '' || $lng === '') { + continue; + } + + $pcd = strtoupper(preg_replace('/\s+/', '', (string) $row[$columns['pcd']])); + + if ($pcd === '' || strlen($pcd) < 5) { + continue; + } + + $buffer[] = [ + 'postcode' => $pcd, + 'outcode' => substr($pcd, 0, strlen($pcd) - 3), + 'lat' => (float) $lat, + 'lng' => (float) $lng, + ]; + + if (count($buffer) >= self::CHUNK_SIZE) { + DB::table('postcodes')->insert($buffer); + $imported += count($buffer); + $buffer = []; + } + } + + if ($buffer !== []) { + DB::table('postcodes')->insert($buffer); + $imported += count($buffer); + } + + fclose($handle); + + $this->info("Imported {$imported} postcodes."); + + return self::SUCCESS; + } +} +``` + +- [ ] **Step 5: Run the test — expect PASS** + +Run: `timeout 10 php artisan test --compact --filter='imports active postcodes'` +Expected: PASS. Two rows in `postcodes`. + +- [ ] **Step 6: Commit** + +```bash +git add app/Console/Commands/ImportPostcodes.php tests/Feature/Console/ImportPostcodesTest.php +git commit -m "feat: add postcodes:import command for loading ONSPD CSV" +``` + +--- + +### Task 8: Import command skips terminated rows and blank coordinates + +**Files:** +- Modify: `tests/Feature/Console/ImportPostcodesTest.php` + +- [ ] **Step 1: Add failing tests for skip behaviour** + +Append to `tests/Feature/Console/ImportPostcodesTest.php`: + +```php +it('skips terminated postcodes', function (): void { + $csv = <<artisan('postcodes:import', ['--file' => $path])->assertSuccessful(); + + expect(Postcode::count())->toBe(1) + ->and(Postcode::find('OLD1AA'))->toBeNull(); +}); + +it('skips rows with blank coordinates', function (): void { + $csv = <<artisan('postcodes:import', ['--file' => $path])->assertSuccessful(); + + expect(Postcode::count())->toBe(1) + ->and(Postcode::find('BT11AA'))->toBeNull(); +}); +``` + +- [ ] **Step 2: Run tests — expect PASS (filters already coded in Task 7)** + +Run: `timeout 10 php artisan test --compact --filter=ImportPostcodes` +Expected: PASS. If either test fails, revisit the filter logic in Task 7 before continuing. + +- [ ] **Step 3: Commit** + +```bash +git add tests/Feature/Console/ImportPostcodesTest.php +git commit -m "test: cover terminated + blank-coord skip paths for postcodes:import" +``` + +--- + +### Task 9: Derive outcode centroids during import + +**Files:** +- Modify: `app/Console/Commands/ImportPostcodes.php` +- Modify: `tests/Feature/Console/ImportPostcodesTest.php` + +- [ ] **Step 1: Add failing test — outcodes populated with averaged coords** + +Append to `tests/Feature/Console/ImportPostcodesTest.php`: + +```php +it('derives outcode centroids as the average of member postcodes', function (): void { + $csv = <<artisan('postcodes:import', ['--file' => $path])->assertSuccessful(); + + $pe7 = Outcode::find('PE7'); + + expect($pe7)->not->toBeNull() + ->and(round((float) $pe7->lat, 6))->toBe(52.550000) + ->and(round((float) $pe7->lng, 6))->toBe(-0.210000) + ->and(Outcode::find('M1'))->not->toBeNull(); +}); +``` + +- [ ] **Step 2: Run test — expect FAIL** + +Run: `timeout 10 php artisan test --compact --filter='derives outcode centroids'` +Expected: FAIL — outcodes table empty. + +- [ ] **Step 3: Populate outcodes after postcode insert** + +In `app/Console/Commands/ImportPostcodes.php`, after the `fclose($handle);` line and before `$this->info(...)`, insert: + +```php + DB::statement( + 'INSERT INTO outcodes (outcode, lat, lng) + SELECT outcode, AVG(lat), AVG(lng) FROM postcodes GROUP BY outcode' + ); + + $this->info('Derived '.DB::table('outcodes')->count().' outcode centroids.'); +``` + +- [ ] **Step 4: Run test — expect PASS** + +Run: `timeout 10 php artisan test --compact --filter='derives outcode centroids'` +Expected: PASS. + +- [ ] **Step 5: Run full ImportPostcodes test file** + +Run: `timeout 10 php artisan test --compact --filter=ImportPostcodes` +Expected: all four tests pass. + +- [ ] **Step 6: Commit** + +```bash +git add app/Console/Commands/ImportPostcodes.php tests/Feature/Console/ImportPostcodesTest.php +git commit -m "feat: derive outcode centroids from postcodes during import" +``` + +--- + +### Task 10: Add ONS / Royal Mail / Ordnance Survey attribution + +**Files:** +- Modify: `resources/js/views/Home.vue` + +The Open Government Licence v3 requires acknowledgement of ONS, Royal Mail, and Ordnance Survey when using ONSPD data. + +- [ ] **Step 1: Add the attribution line in the footer** + +In `resources/js/views/Home.vue`, locate the footer bottom-bar block that contains: + +```html +

Data provided by official UK retail price transparency schemes.

+``` + +Directly below that `

`, add a second line: + +```html +

Postcode data contains OS data © Crown copyright & database right, Royal Mail data © Royal Mail copyright & database right, and National Statistics data © Crown copyright & database right.

+``` + +- [ ] **Step 2: Visually check the footer** + +Run: `npm run dev` and load the home page in the browser. Confirm the attribution line is visible under the data-provider line. There is nothing to test programmatically here. + +- [ ] **Step 3: Commit** + +```bash +git add resources/js/views/Home.vue +git commit -m "docs: add ONS/Royal Mail/OS attribution required by OGL v3" +``` + +--- + +### Task 11: Run the real import end-to-end (manual) + +This step is not a coding task — it is how you actually populate the table. + +- [ ] **Step 1: Download ONSPD** + +Go to the ONS Open Geography Portal and download the latest ONSPD release as a ZIP (~350MB). Extract the single-file CSV `Data/ONSPD__UK.csv`. + +- [ ] **Step 2: Run the import** + +Run: +```bash +php artisan postcodes:import --file=/absolute/path/to/ONSPD_MMM_YYYY_UK.csv +``` +Expected: takes 30–90s; reports `Imported ~2,600,000 postcodes.` and `Derived ~2,900 outcode centroids.` (numbers approximate). + +- [ ] **Step 3: Spot-check via tinker** + +Run: +```bash +php artisan tinker --execute 'echo App\Models\Postcode::count();' +php artisan tinker --execute 'print_r(App\Models\Postcode::find("SW1A1AA")?->toArray());' +php artisan tinker --execute 'print_r(App\Models\Outcode::find("PE7")?->toArray());' +``` +Expected: row counts in the millions / thousands; SW1A1AA resolves to ~(51.501, -0.142); PE7 resolves to a centroid near Peterborough. + +- [ ] **Step 4: Format PHP** + +Run: `vendor/bin/pint --dirty --format agent` + +- [ ] **Step 5: Final full-suite check** + +Run: `php artisan test --compact` +Expected: all tests green. + +--- + +## Self-Review Notes + +- **Spec coverage:** migrations ✓, models ✓, local DB lookup for full postcode ✓ and outcode ✓, HTTP fallback persistence ✓, cache re-scoping ✓, import command + terminated/blank filter ✓, outcode centroid derivation ✓, attribution ✓, end-to-end run ✓. +- **Method signatures are consistent:** `lookupLocalPostcode`, `lookupLocalOutcode`, `formatPostcode` appear exactly with those names in every task that references them. +- **Idempotency:** `postcodes:import` truncates both tables before inserting, so it is safe to re-run for quarterly refreshes. +- **No scheduler automation in v1:** ONSPD is quarterly and requires manual download (license click-through) — the command is operator-run by design. A scheduled auto-fetch can be added later if the user wants it. diff --git a/resources/js/views/Home.vue b/resources/js/views/Home.vue index af06066..b3029f8 100644 --- a/resources/js/views/Home.vue +++ b/resources/js/views/Home.vue @@ -380,6 +380,7 @@

© 2024 FuelAlert UK Limited. All Rights Reserved.

Data provided by official UK retail price transparency schemes.

+

Postcode data contains OS data © Crown copyright & database right, Royal Mail data © Royal Mail copyright & database right, and National Statistics data © Crown copyright & database right.

diff --git a/tests/Feature/Console/ImportPostcodesTest.php b/tests/Feature/Console/ImportPostcodesTest.php new file mode 100644 index 0000000..3d60e68 --- /dev/null +++ b/tests/Feature/Console/ImportPostcodesTest.php @@ -0,0 +1,84 @@ +artisan('postcodes:import', ['--file' => $path]) + ->assertSuccessful(); + + expect(Postcode::count())->toBe(2) + ->and(Postcode::find('SW1A1AA')->outcode)->toBe('SW1A') + ->and(Postcode::find('M11AD')->outcode)->toBe('M1'); +}); + +it('skips terminated postcodes', function (): void { + $csv = <<<'CSV' +pcd,pcds,doterm,lat,long +"SW1A1AA","SW1A 1AA","",51.501009,-0.141588 +"OLD1AA","OLD 1AA","202301",50.000000,-1.000000 +CSV; + + $path = writeOnspdFixture($csv); + + $this->artisan('postcodes:import', ['--file' => $path])->assertSuccessful(); + + expect(Postcode::count())->toBe(1) + ->and(Postcode::find('OLD1AA'))->toBeNull(); +}); + +it('skips rows with blank coordinates', function (): void { + $csv = <<<'CSV' +pcd,pcds,doterm,lat,long +"SW1A1AA","SW1A 1AA","",51.501009,-0.141588 +"BT11AA","BT1 1AA","",, +CSV; + + $path = writeOnspdFixture($csv); + + $this->artisan('postcodes:import', ['--file' => $path])->assertSuccessful(); + + expect(Postcode::count())->toBe(1) + ->and(Postcode::find('BT11AA'))->toBeNull(); +}); + +it('derives outcode centroids as the average of member postcodes', function (): void { + $csv = <<<'CSV' +pcd,pcds,doterm,lat,long +"PE71AA","PE7 1AA","",52.500000,-0.200000 +"PE71AB","PE7 1AB","",52.600000,-0.220000 +"M11AD","M1 1AD","",53.480957,-2.237428 +CSV; + + $path = writeOnspdFixture($csv); + + $this->artisan('postcodes:import', ['--file' => $path])->assertSuccessful(); + + $pe7 = Outcode::find('PE7'); + + expect($pe7)->not->toBeNull() + ->and(round((float) $pe7->lat, 6))->toBe(52.550000) + ->and(round((float) $pe7->lng, 6))->toBe(-0.210000) + ->and(Outcode::find('M1'))->not->toBeNull(); +}); diff --git a/tests/Unit/Services/PostcodeServiceTest.php b/tests/Unit/Services/PostcodeServiceTest.php index 2417240..9e99cfc 100644 --- a/tests/Unit/Services/PostcodeServiceTest.php +++ b/tests/Unit/Services/PostcodeServiceTest.php @@ -1,5 +1,7 @@ Http::response([ + '*/places*' => Http::response([ + 'status' => 200, + 'result' => [[ + 'name_1' => 'Manchester', + 'latitude' => 53.480957, + 'longitude' => -2.237428, + ]], + ]), + ]); + + $this->service->resolve('Manchester'); + $this->service->resolve('Manchester'); + + Http::assertSentCount(1); + expect(Cache::get('place:manchester'))->toBeInstanceOf(LocationResult::class); +}); + +it('does not cache postcode resolutions in the Cache store (DB is the cache)', function (): void { + Http::fake([ + '*/postcodes/SW1A1AA' => Http::response([ 'status' => 200, 'result' => [ - 'outcode' => 'PE7', - 'latitude' => 52.536397, - 'longitude' => -0.210181, + 'postcode' => 'SW1A 1AA', + 'latitude' => 51.501009, + 'longitude' => -0.141588, ], ]), ]); - $this->service->resolve('PE7'); - $this->service->resolve('PE7'); + $this->service->resolve('SW1A 1AA'); - Http::assertSentCount(1); - expect(Cache::get('postcode:pe7'))->toBeInstanceOf(LocationResult::class); + expect(Cache::get('postcode:sw1a1aa'))->toBeNull() + ->and(Postcode::find('SW1A1AA'))->not->toBeNull(); }); it('does not cache failed lookups', function (): void { @@ -171,3 +191,91 @@ it('returns null and does not throw on API failure', function (): void { expect($result)->toBeNull(); }); + +// --- Local DB (full postcode) --- + +it('resolves a full postcode from local DB without calling HTTP', function (): void { + Postcode::create([ + 'postcode' => 'SW1A1AA', + 'outcode' => 'SW1A', + 'lat' => 51.501009, + 'lng' => -0.141588, + ]); + + Http::fake(); // any HTTP call will be recorded + + $result = $this->service->resolve('SW1A 1AA'); + + expect($result)->toBeInstanceOf(LocationResult::class) + ->and($result->displayName)->toBe('SW1A 1AA') + ->and($result->lat)->toBe(51.501009) + ->and($result->lng)->toBe(-0.141588); + + Http::assertNothingSent(); +}); + +// --- Local DB (outcode) --- + +it('resolves an outcode from local DB without calling HTTP', function (): void { + Outcode::create([ + 'outcode' => 'PE7', + 'lat' => 52.536397, + 'lng' => -0.210181, + ]); + + Http::fake(); + + $result = $this->service->resolve('PE7'); + + expect($result)->toBeInstanceOf(LocationResult::class) + ->and($result->displayName)->toBe('PE7') + ->and($result->lat)->toBe(52.536397) + ->and($result->lng)->toBe(-0.210181); + + Http::assertNothingSent(); +}); + +// --- HTTP fallback persistence --- + +it('persists a full postcode resolved via HTTP fallback', function (): void { + Http::fake([ + '*/postcodes/SW1A1AA' => Http::response([ + 'status' => 200, + 'result' => [ + 'postcode' => 'SW1A 1AA', + 'latitude' => 51.501009, + 'longitude' => -0.141588, + ], + ]), + ]); + + $this->service->resolve('SW1A 1AA'); + + $row = Postcode::find('SW1A1AA'); + + expect($row)->not->toBeNull() + ->and($row->outcode)->toBe('SW1A') + ->and((float) $row->lat)->toBe(51.501009) + ->and((float) $row->lng)->toBe(-0.141588); +}); + +it('persists an outcode resolved via HTTP fallback', function (): void { + Http::fake([ + '*/outcodes/PE7' => Http::response([ + 'status' => 200, + 'result' => [ + 'outcode' => 'PE7', + 'latitude' => 52.536397, + 'longitude' => -0.210181, + ], + ]), + ]); + + $this->service->resolve('PE7'); + + $row = Outcode::find('PE7'); + + expect($row)->not->toBeNull() + ->and((float) $row->lat)->toBe(52.536397) + ->and((float) $row->lng)->toBe(-0.210181); +});