3 - Harden fediverse polling: timeouts, error handling, payload fields

database/factories/PageFactory.php

@@ -1,24 +0,0 @@
-<?php
-
-namespace Database\Factories;
-
-use App\Models\Page;
-use Illuminate\Database\Eloquent\Factories\Factory;
-
-/**
- * @extends Factory<Page>
- */
-class PageFactory extends Factory
-{
-    /**
-     * Define the model's default state.
-     *
-     * @return array<string, mixed>
-     */
-    public function definition(): array
-    {
-        return [
-            //
-        ];
-    }
-}

packages/Lvl0/FediDiscover/src/Actions/PollFediverseAction.php

@@ -4,10 +4,13 @@
 
 namespace Lvl0\FediDiscover\Actions;
 
+use Carbon\CarbonImmutable;
+use Illuminate\Support\Facades\Log;
 use Lvl0\FediDiscover\Clients\FediverseClientFactory;
 use Lvl0\FediDiscover\Events\UrlDiscovered;
 use Lvl0\FediDiscover\Models\Instance;
 use Lvl0\FediDiscover\ValueObjects\FediversePost;
+use Throwable;
 
 class PollFediverseAction
 {
@@ -19,7 +22,17 @@ public function execute(Instance $instance): void
         $posts = $client->fetchPostsSince($instance, $instance->last_seen_id);
 
         $posts->each(function (FediversePost $post) use ($instance) {
-            $this->processLinks($post, $instance);
+            try {
+                $this->processLinks($post, $instance);
+            } catch (Throwable $e) {
+                Log::warning('fedi-discover:processLinks failed', [
+                    'instance_id' => $instance->id,
+                    'instance_url' => $instance->url,
+                    'post_url' => $post->selfUrl,
+                    'exception' => $e::class,
+                    'message' => $e->getMessage(),
+                ]);
+            }
         });
 
         if ($posts->isNotEmpty()) {
@@ -49,6 +62,8 @@ private function processLinks(FediversePost $post, Instance $instance): void
             ->unique()
             ->each(fn (string $url) => UrlDiscovered::dispatch(
                 url: $url,
+                instanceId: $instance->id,
+                discoveredAt: CarbonImmutable::now(),
                 postUrl: $post->selfUrl,
                 postBody: $post->body,
             ));

packages/Lvl0/FediDiscover/src/Clients/FediverseClientInterface.php

@@ -6,6 +6,7 @@
 
 use Illuminate\Support\Collection;
 use Lvl0\FediDiscover\Models\Instance;
+use Lvl0\FediDiscover\ValueObjects\FediversePost;
 
 interface FediverseClientInterface
 {

packages/Lvl0/FediDiscover/src/Clients/LemmyClient.php

@@ -19,13 +19,13 @@ public function fetchPostsSince(Instance $instance, ?string $lastSeenId): Collec
 
         $response = Http::withHeaders([
             'User-Agent' => config('fedi-discover.http.user_agent'),
-        ])->get($url, $params);
+        ])->timeout(config('fedi-discover.http.timeout'))->get($url, $params);
 
-        if (! $response->successful() || ! is_array($response->json())) {
+        if (! $response->successful()) {
             return collect();
         }
 
-        return collect($response->json()['posts'])
+        return collect($response->json('posts', []))
             ->map(fn (array $p) => $p['post'])
             ->map(function (array $t) {
                 $parts = array_filter([$t['body'] ?? null, $t['url'] ?? null]);

packages/Lvl0/FediDiscover/src/Clients/MastodonClient.php

@@ -19,16 +19,16 @@ public function fetchPostsSince(Instance $instance, ?string $lastSeenId): Collec
 
         $response = Http::withHeaders([
             'User-Agent' => config('fedi-discover.http.user_agent'),
-        ])->get($url, $params);
+        ])->timeout(config('fedi-discover.http.timeout'))->get($url, $params);
 
-        if (! $response->successful() || ! is_array($response->json())) {
+        if (! $response->successful()) {
             return collect();
         }
 
-        return collect($response->json())
+        return collect($response->json() ?? [])
             ->map(fn (array $t) => new FediversePost(
                 cursorId: $t['id'],
-                selfUrl: $t['url'] ?? $t['uri'],
+                selfUrl: $t['url'] ?? $t['uri'] ?? null,
                 body: $t['content'],
                 publishedAt: $t['created_at'] ?? null
             ));

packages/Lvl0/FediDiscover/src/Console/Commands/PollInstancesCommand.php

@@ -7,6 +7,7 @@
 use Illuminate\Console\Attributes\Description;
 use Illuminate\Console\Attributes\Signature;
 use Illuminate\Console\Command;
+use Illuminate\Support\Facades\Log;
 use Lvl0\FediDiscover\Actions\PollFediverseAction;
 use Lvl0\FediDiscover\Models\Instance;
 use Throwable;
@@ -31,6 +32,13 @@ public function handle(): int
                 try {
                     $this->action->execute($instance);
                 } catch (Throwable $e) {
+                    $this->error("Failed to poll {$instance->url}: {$e->getMessage()}");
+                    Log::warning('fedi-discover:poll failed', [
+                        'instance_id' => $instance->id,
+                        'instance_url' => $instance->url,
+                        'exception' => $e::class,
+                        'message' => $e->getMessage(),
+                    ]);
                     $hadFailure = true;
                 }
             });

packages/Lvl0/FediDiscover/src/Events/UrlDiscovered.php

@@ -1,26 +1,22 @@
 <?php
 
+declare(strict_types=1);
+
 namespace Lvl0\FediDiscover\Events;
 
-use Illuminate\Broadcasting\InteractsWithSockets;
+use Carbon\CarbonImmutable;
-use Illuminate\Broadcasting\PrivateChannel;
 use Illuminate\Foundation\Events\Dispatchable;
 use Illuminate\Queue\SerializesModels;
 
 class UrlDiscovered
 {
-    use Dispatchable, InteractsWithSockets, SerializesModels;
+    use Dispatchable, SerializesModels;
 
     public function __construct(
         public string $url,
+        public int $instanceId,
+        public CarbonImmutable $discoveredAt,
         public ?string $postUrl = null,
-        public ?string $postBody = null
+        public ?string $postBody = null,
     ) {}
-
-    public function broadcastOn(): array
-    {
-        return [
-            new PrivateChannel('discovery'),
-        ];
-    }
 }

packages/Lvl0/FediDiscover/src/FediDiscoverServiceProvider.php

@@ -4,12 +4,10 @@
 
 namespace Lvl0\FediDiscover;
 
-use Illuminate\Support\Facades\Event;
 use Illuminate\Support\ServiceProvider;
 use Lvl0\FediDiscover\Clients\FediverseClientFactory;
 use Lvl0\FediDiscover\Console\Commands\PollInstancesCommand;
 use Lvl0\FediDiscover\Console\Commands\ValidateInstancesCommand;
-use Lvl0\FediDiscover\Events\UrlDiscovered;
 
 class FediDiscoverServiceProvider extends ServiceProvider
 {
@@ -24,10 +22,6 @@ public function boot(): void
     {
         $this->loadMigrationsFrom(__DIR__ . '/../database/migrations');
 
-        Event::listen(
-            UrlDiscovered::class,
-        );
-
         if ($this->app->runningInConsole()) {
             $this->publishes([
                 __DIR__ . '/../config/fedi-discover.php' => config_path('fedi-discover.php'),

packages/Lvl0/FediDiscover/src/Models/Instance.php

@@ -40,7 +40,11 @@ class Instance extends Model
         'last_polled_at' => 'datetime',
     ];
 
-    public function scopeEnabled($query): Builder
+    /**
+     * @param  Builder<self>  $query
+     * @return Builder<self>
+     */
+    public function scopeEnabled(Builder $query): Builder
     {
         return $query->where('enabled', true);
     }

packages/Lvl0/FediDiscover/src/ValueObjects/FediversePost.php

@@ -8,7 +8,7 @@ class FediversePost
 {
     public function __construct(
         public string $cursorId,
-        public string $selfUrl,
+        public ?string $selfUrl,
         public ?string $body = null,
         public ?string $title = null,
         public ?string $publishedAt = null,

packages/Lvl0/FediDiscover/tests/Feature/PollFediverseActionTest.php

@@ -4,6 +4,7 @@
 
 namespace Lvl0\FediDiscover\Tests\Feature;
 
+use Carbon\CarbonImmutable;
 use Illuminate\Foundation\Testing\RefreshDatabase;
 use Illuminate\Support\Facades\Event;
 use Lvl0\FediDiscover\Actions\PollFediverseAction;
@@ -119,13 +120,17 @@ public function test_it_passes_post_self_url_and_body_through_to_the_event(): vo
     {
         Event::fake([UrlDiscovered::class]);
 
+        $instance = $this->makeInstance();
         $body = 'Here is https://example.com/article with surrounding context.';
 
-        $this->poll([
+        $this->pollInstance($instance, [
             new FediversePost('1', 'https://mastodon.social/@alice/1', $body),
         ]);
 
-        Event::assertDispatched(UrlDiscovered::class, fn (UrlDiscovered $e) => $e->postUrl === 'https://mastodon.social/@alice/1' && $e->postBody === $body
+        Event::assertDispatched(UrlDiscovered::class, fn (UrlDiscovered $e) => $e->postUrl === 'https://mastodon.social/@alice/1'
+            && $e->postBody === $body
+            && $e->instanceId === $instance->id
+            && $e->discoveredAt instanceof CarbonImmutable
         );
     }
 

packages/Lvl0/FediDiscover/tests/Unit/UrlDiscoveredTest.php

@@ -4,6 +4,7 @@
 
 namespace Lvl0\FediDiscover\Tests\Unit;
 
+use Carbon\CarbonImmutable;
 use Lvl0\FediDiscover\Events\UrlDiscovered;
 use PHPUnit\Framework\TestCase;
 
@@ -11,13 +12,19 @@ class UrlDiscoveredTest extends TestCase
 {
     public function test_it_exposes_all_payload_fields(): void
     {
+        $discoveredAt = CarbonImmutable::parse('2026-04-26T12:00:00');
+
         $event = new UrlDiscovered(
             url: 'https://example.com/article',
+            instanceId: 42,
+            discoveredAt: $discoveredAt,
             postUrl: 'https://mastodon.social/@alice/109876543210',
             postBody: 'Check out this article: https://example.com/article'
         );
 
         $this->assertSame('https://example.com/article', $event->url);
+        $this->assertSame(42, $event->instanceId);
+        $this->assertTrue($discoveredAt->eq($event->discoveredAt));
         $this->assertSame('https://mastodon.social/@alice/109876543210', $event->postUrl);
         $this->assertSame('Check out this article: https://example.com/article', $event->postBody);
     }
@@ -26,6 +33,8 @@ public function test_post_body_is_nullable(): void
     {
         $event = new UrlDiscovered(
             url: 'https://example.com/article',
+            instanceId: 1,
+            discoveredAt: CarbonImmutable::parse('2026-04-26T12:00:00'),
             postUrl: 'https://mastodon.social/@alice/109876543210',
             postBody: null
         );