95 - Add a platform credential health check

app/Actions/CreatePlatformAccountAction.php

@@ -2,6 +2,7 @@
 
 namespace App\Actions;
 
+use App\Enums\AccountStatusEnum;
 use App\Exceptions\PlatformAuthException;
 use App\Models\PlatformAccount;
 use App\Models\PlatformInstance;
@@ -46,7 +47,7 @@ public function execute(string $instanceDomain, string $username, string $passwo
                     'api_token' => $authResponse['jwt'] ?? null,
                 ],
                 'is_active' => true,
-                'status' => 'active',
+                'status' => AccountStatusEnum::HEALTHY,
             ]);
         });
     }

app/Enums/AccountStatusEnum.php

@@ -0,0 +1,19 @@
+<?php
+
+namespace App\Enums;
+
+enum AccountStatusEnum: string
+{
+    case UNTESTED = 'untested';
+    case HEALTHY = 'healthy';
+    case UNHEALTHY = 'unhealthy';
+
+    public function label(): string
+    {
+        return match ($this) {
+            self::UNTESTED => 'Untested',
+            self::HEALTHY => 'Healthy',
+            self::UNHEALTHY => 'Unhealthy',
+        };
+    }
+}

app/Jobs/CheckPlatformCredentialsJob.php

@@ -0,0 +1,79 @@
+<?php
+
+namespace App\Jobs;
+
+use App\Enums\NotificationSeverityEnum;
+use App\Enums\NotificationTypeEnum;
+use App\Models\Notification;
+use App\Models\PlatformAccount;
+use App\Modules\Lemmy\Services\LemmyApiService;
+use App\Services\Notification\NotificationService;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Queue\Queueable;
+use Throwable;
+
+class CheckPlatformCredentialsJob implements ShouldQueue
+{
+    use Queueable;
+
+    public function handle(NotificationService $notificationService): void
+    {
+        $accounts = PlatformAccount::where('is_active', true)->get();
+
+        foreach ($accounts as $account) {
+            $this->check($account, $notificationService);
+        }
+    }
+
+    private function check(PlatformAccount $account, NotificationService $notificationService): void
+    {
+        if ($this->canLogIn($account)) {
+            $account->recordCredentialCheckPassed();
+
+            return;
+        }
+
+        $wasUnhealthy = $account->isUnhealthy();
+        $account->recordCredentialCheckFailed();
+
+        if (! $wasUnhealthy && $account->refresh()->isUnhealthy()) {
+            $this->notify($account, $notificationService);
+        }
+    }
+
+    private function canLogIn(PlatformAccount $account): bool
+    {
+        try {
+            return $this->makeApiService($account)->login($account->username, $account->password) !== null;
+        } catch (Throwable) {
+            return false;
+        }
+    }
+
+    protected function makeApiService(PlatformAccount $account): LemmyApiService
+    {
+        return new LemmyApiService($account->instance_url);
+    }
+
+    private function notify(PlatformAccount $account, NotificationService $notificationService): void
+    {
+        $alreadyNotified = Notification::query()
+            ->where('type', NotificationTypeEnum::CREDENTIAL_EXPIRED)
+            ->where('notifiable_type', $account->getMorphClass())
+            ->where('notifiable_id', $account->getKey())
+            ->unread()
+            ->exists();
+
+        if ($alreadyNotified) {
+            return;
+        }
+
+        $notificationService->send(
+            type: NotificationTypeEnum::CREDENTIAL_EXPIRED,
+            severity: NotificationSeverityEnum::ERROR,
+            title: "Credentials failed for {$account->username}",
+            message: "Could not log in to {$account->instance_url} after ".PlatformAccount::FAILURES_BEFORE_UNHEALTHY.' attempts. Publishing to this account will fail until it is fixed.',
+            notifiable: $account,
+        );
+    }
+}

app/Models/PlatformAccount.php

@@ -2,6 +2,7 @@
 
 namespace App\Models;
 
+use App\Enums\AccountStatusEnum;
 use App\Enums\PlatformEnum;
 use Database\Factories\PlatformAccountFactory;
 use Illuminate\Database\Eloquent\Casts\Attribute;
@@ -21,7 +22,8 @@
  * @property array<string, mixed> $settings
  * @property bool $is_active
  * @property Carbon|null $last_tested_at
- * @property string $status
+ * @property AccountStatusEnum $status
+ * @property int $consecutive_failures
  * @property Carbon $created_at
  * @property Carbon $updated_at
  * @property Collection<int, PlatformChannel> $activeChannels
@@ -44,10 +46,12 @@ class PlatformAccount extends Model
         'is_active',
         'last_tested_at',
         'status',
+        'consecutive_failures',
     ];
 
     protected $casts = [
         'platform' => PlatformEnum::class,
+        'status' => AccountStatusEnum::class,
         'settings' => 'array',
         'is_active' => 'boolean',
         'last_tested_at' => 'datetime',
@@ -136,4 +140,33 @@ public function activeChannels(): BelongsToMany
             ->wherePivot('is_active', true)
             ->orderByPivot('priority', 'desc');
     }
+
+    public const FAILURES_BEFORE_UNHEALTHY = 3;
+
+    public function recordCredentialCheckPassed(): void
+    {
+        $this->update([
+            'status' => AccountStatusEnum::HEALTHY,
+            'consecutive_failures' => 0,
+            'last_tested_at' => now(),
+        ]);
+    }
+
+    public function recordCredentialCheckFailed(): void
+    {
+        $failures = $this->consecutive_failures + 1;
+
+        $this->update([
+            'status' => $failures >= self::FAILURES_BEFORE_UNHEALTHY
+                ? AccountStatusEnum::UNHEALTHY
+                : $this->status,
+            'consecutive_failures' => $failures,
+            'last_tested_at' => now(),
+        ]);
+    }
+
+    public function isUnhealthy(): bool
+    {
+        return $this->status === AccountStatusEnum::UNHEALTHY;
+    }
 }

database/factories/PlatformAccountFactory.php

@@ -2,6 +2,7 @@
 
 namespace Database\Factories;
 
+use App\Enums\AccountStatusEnum;
 use App\Enums\PlatformEnum;
 use App\Models\PlatformAccount;
 use Illuminate\Database\Eloquent\Factories\Factory;
@@ -23,7 +24,7 @@ public function definition(): array
             'settings' => [],
             'is_active' => true,
             'last_tested_at' => null,
-            'status' => 'untested',
+            'status' => AccountStatusEnum::UNTESTED,
         ];
     }
 
@@ -38,7 +39,7 @@ public function tested(): static
     {
         return $this->state(fn (array $attributes) => [
             'last_tested_at' => now()->subHours(2),
-            'status' => 'working',
+            'status' => AccountStatusEnum::HEALTHY,
         ]);
     }
 
@@ -46,7 +47,8 @@ public function failed(): static
     {
         return $this->state(fn (array $attributes) => [
             'last_tested_at' => now()->subHours(2),
-            'status' => 'failed',
+            'status' => AccountStatusEnum::UNHEALTHY,
+            'consecutive_failures' => PlatformAccount::FAILURES_BEFORE_UNHEALTHY,
         ]);
     }
 }

database/migrations/2024_01_01_000020_add_credential_health_to_platform_accounts.php

@@ -0,0 +1,28 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\DB;
+use Illuminate\Support\Facades\Schema;
+
+return new class extends Migration
+{
+    public function up(): void
+    {
+        Schema::table('platform_accounts', function (Blueprint $table) {
+            $table->unsignedTinyInteger('consecutive_failures')->default(0)->after('status');
+        });
+
+        DB::table('platform_accounts')->where('status', 'active')->update(['status' => 'healthy']);
+    }
+
+    public function down(): void
+    {
+        // Not a true inverse: accounts the health check marked healthy also become 'active'.
+        DB::table('platform_accounts')->where('status', 'healthy')->update(['status' => 'active']);
+
+        Schema::table('platform_accounts', function (Blueprint $table) {
+            $table->dropColumn('consecutive_failures');
+        });
+    }
+};

resources/views/livewire/channels.blade.php

@@ -69,7 +69,14 @@ class="text-sm text-blue-600 hover:text-blue-800 dark:text-blue-400"
                         <div class="space-y-1">
                             @foreach ($channel->platformAccounts->take(3) as $account)
                                 <div class="flex items-center justify-between text-sm">
-                                    <span class="text-gray-600 dark:text-gray-300">{{ $account->username }}</span>
+                                    <span class="flex items-center gap-x-1.5">
+                                        <span class="text-gray-600 dark:text-gray-300">{{ $account->username }}</span>
+                                        @if ($account->isUnhealthy())
+                                            <span class="inline-flex items-center px-1.5 py-0.5 rounded-full text-xs font-medium bg-red-100 text-red-800 dark:bg-red-900/30 dark:text-red-300" title="Login failed on the last {{ App\Models\PlatformAccount::FAILURES_BEFORE_UNHEALTHY }} checks">
+                                                Credentials failing
+                                            </span>
+                                        @endif
+                                    </span>
                                     <button
                                         wire:click="detachAccount({{ $channel->id }}, {{ $account->id }})"
                                         class="text-red-500 hover:text-red-700"

routes/console.php

@@ -2,6 +2,7 @@
 
 use App\Jobs\ArticleDiscoveryJob;
 use App\Jobs\CheckFeedStalenessJob;
+use App\Jobs\CheckPlatformCredentialsJob;
 use App\Jobs\CleanupActivityLogsJob;
 use App\Jobs\CleanupArticlesJob;
 use App\Jobs\PublishNextArticleJob;
@@ -30,6 +31,12 @@
     ->withoutOverlapping()
     ->onOneServer();
 
+Schedule::job(new CheckPlatformCredentialsJob)
+    ->daily()
+    ->name('check-platform-credentials')
+    ->withoutOverlapping()
+    ->onOneServer();
+
 Schedule::job(new CleanupArticlesJob)
     ->daily()
     ->name('cleanup-old-articles')

tests/Feature/Jobs/CheckPlatformCredentialsJobTest.php

@@ -0,0 +1,163 @@
+<?php
+
+namespace Tests\Feature\Jobs;
+
+use App\Enums\AccountStatusEnum;
+use App\Enums\NotificationTypeEnum;
+use App\Jobs\CheckPlatformCredentialsJob;
+use App\Models\Notification;
+use App\Models\PlatformAccount;
+use App\Modules\Lemmy\Services\LemmyApiService;
+use App\Services\Notification\NotificationService;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+use Mockery;
+use Tests\TestCase;
+
+class CheckPlatformCredentialsJobTest extends TestCase
+{
+    use RefreshDatabase;
+
+    private function runJob(bool $loginSucceeds): void
+    {
+        $api = Mockery::mock(LemmyApiService::class);
+        $api->shouldReceive('login')->andReturn($loginSucceeds ? 'a-token' : null);
+
+        $job = new class($api) extends CheckPlatformCredentialsJob
+        {
+            public function __construct(private LemmyApiService $api) {}
+
+            protected function makeApiService(PlatformAccount $account): LemmyApiService
+            {
+                return $this->api;
+            }
+        };
+
+        $job->handle(app(NotificationService::class));
+    }
+
+    /**
+     * @param  array<string, mixed>  $attributes
+     */
+    private function account(array $attributes = []): PlatformAccount
+    {
+        return PlatformAccount::factory()->create(array_merge(['is_active' => true], $attributes));
+    }
+
+    public function test_a_successful_login_marks_the_account_healthy(): void
+    {
+        $account = $this->account(['status' => AccountStatusEnum::UNTESTED]);
+
+        $this->runJob(true);
+
+        $account->refresh();
+        $this->assertSame(AccountStatusEnum::HEALTHY, $account->status);
+        $this->assertSame(0, $account->consecutive_failures);
+        $this->assertNotNull($account->last_tested_at);
+    }
+
+    public function test_a_successful_login_clears_earlier_failures(): void
+    {
+        $account = $this->account(['status' => AccountStatusEnum::UNHEALTHY, 'consecutive_failures' => 2]);
+
+        $this->runJob(true);
+
+        $account->refresh();
+        $this->assertSame(AccountStatusEnum::HEALTHY, $account->status);
+        $this->assertSame(0, $account->consecutive_failures);
+    }
+
+    public function test_a_single_failure_does_not_mark_the_account_unhealthy(): void
+    {
+        $account = $this->account(['status' => AccountStatusEnum::HEALTHY]);
+
+        $this->runJob(false);
+
+        $account->refresh();
+        $this->assertSame(AccountStatusEnum::HEALTHY, $account->status);
+        $this->assertSame(1, $account->consecutive_failures);
+    }
+
+    public function test_the_threshold_failure_marks_the_account_unhealthy(): void
+    {
+        $account = $this->account([
+            'status' => AccountStatusEnum::HEALTHY,
+            'consecutive_failures' => PlatformAccount::FAILURES_BEFORE_UNHEALTHY - 1,
+        ]);
+
+        $this->runJob(false);
+
+        $this->assertSame(AccountStatusEnum::UNHEALTHY, $account->refresh()->status);
+    }
+
+    public function test_a_single_failure_does_not_notify(): void
+    {
+        $this->account(['status' => AccountStatusEnum::HEALTHY]);
+
+        $this->runJob(false);
+
+        $this->assertDatabaseCount('notifications', 0);
+    }
+
+    public function test_becoming_unhealthy_notifies(): void
+    {
+        $account = $this->account([
+            'status' => AccountStatusEnum::HEALTHY,
+            'consecutive_failures' => PlatformAccount::FAILURES_BEFORE_UNHEALTHY - 1,
+            'username' => 'newsbot',
+        ]);
+
+        $this->runJob(false);
+
+        $this->assertDatabaseHas('notifications', [
+            'type' => NotificationTypeEnum::CREDENTIAL_EXPIRED->value,
+            'notifiable_type' => $account->getMorphClass(),
+            'notifiable_id' => $account->id,
+        ]);
+        $this->assertStringContainsString('newsbot', Notification::first()->title);
+    }
+
+    public function test_an_account_already_unhealthy_does_not_notify_again(): void
+    {
+        $this->account([
+            'status' => AccountStatusEnum::UNHEALTHY,
+            'consecutive_failures' => PlatformAccount::FAILURES_BEFORE_UNHEALTHY,
+        ]);
+
+        $this->runJob(false);
+
+        $this->assertDatabaseCount('notifications', 0);
+    }
+
+    public function test_inactive_accounts_are_not_checked(): void
+    {
+        $account = $this->account(['is_active' => false, 'status' => AccountStatusEnum::UNTESTED]);
+
+        $this->runJob(false);
+
+        $account->refresh();
+        $this->assertSame(AccountStatusEnum::UNTESTED, $account->status);
+        $this->assertNull($account->last_tested_at);
+    }
+
+    public function test_an_exception_during_login_counts_as_a_failure(): void
+    {
+        $account = $this->account(['status' => AccountStatusEnum::HEALTHY]);
+
+        $api = Mockery::mock(LemmyApiService::class);
+        $api->shouldReceive('login')->andThrow(new \RuntimeException('instance unreachable'));
+
+        $job = new class($api) extends CheckPlatformCredentialsJob
+        {
+            public function __construct(private LemmyApiService $api) {}
+
+            protected function makeApiService(PlatformAccount $account): LemmyApiService
+            {
+                return $this->api;
+            }
+        };
+
+        $job->handle(app(NotificationService::class));
+
+        $this->assertSame(1, $account->refresh()->consecutive_failures);
+    }
+}

tests/Unit/Actions/CreatePlatformAccountActionTest.php

@@ -3,6 +3,7 @@
 namespace Tests\Unit\Actions;
 
 use App\Actions\CreatePlatformAccountAction;
+use App\Enums\AccountStatusEnum;
 use App\Enums\PlatformEnum;
 use App\Exceptions\PlatformAuthException;
 use App\Models\PlatformAccount;
@@ -54,7 +55,7 @@ public function test_creates_platform_account_with_new_instance(): void
         $this->assertEquals('https://lemmy.world', $account->instance_url);
         $this->assertEquals('lemmy', $account->platform->value);
         $this->assertTrue($account->is_active);
-        $this->assertEquals('active', $account->status);
+        $this->assertSame(AccountStatusEnum::HEALTHY, $account->status);
         $this->assertEquals(42, $account->settings['person_id']);
         $this->assertEquals('Test User', $account->settings['display_name']);
         $this->assertEquals('A test bio', $account->settings['description']);

tests/Unit/Models/PlatformAccountTest.php

@@ -2,6 +2,7 @@
 
 namespace Tests\Unit\Models;
 
+use App\Enums\AccountStatusEnum;
 use App\Enums\PlatformEnum;
 use App\Models\PlatformAccount;
 use App\Models\PlatformChannel;
@@ -15,7 +16,7 @@ class PlatformAccountTest extends TestCase
 
     public function test_fillable_fields(): void
     {
-        $fillableFields = ['platform', 'instance_url', 'username', 'password', 'settings', 'is_active', 'last_tested_at', 'status'];
+        $fillableFields = ['platform', 'instance_url', 'username', 'password', 'settings', 'is_active', 'last_tested_at', 'status', 'consecutive_failures'];
         $account = new PlatformAccount;
 
         $this->assertEquals($fillableFields, $account->getFillable());
@@ -253,7 +254,7 @@ public function test_account_creation_with_factory(): void
         $this->assertEquals('test-password', $account->password);
         $this->assertIsBool($account->is_active);
         $this->assertTrue($account->is_active);
-        $this->assertEquals('untested', $account->status);
+        $this->assertSame(AccountStatusEnum::UNTESTED, $account->status);
         $this->assertIsArray($account->settings);
     }
 
@@ -270,7 +271,7 @@ public function test_account_creation_with_explicit_values(): void
             'settings' => $settings,
             'is_active' => false,
             'last_tested_at' => $timestamp,
-            'status' => 'working',
+            'status' => AccountStatusEnum::HEALTHY,
         ]);
 
         $this->assertEquals(PlatformEnum::LEMMY, $account->platform);
@@ -280,7 +281,7 @@ public function test_account_creation_with_explicit_values(): void
         $this->assertEquals($settings, $account->settings);
         $this->assertFalse($account->is_active);
         $this->assertEquals($timestamp->format('Y-m-d H:i:s'), $account->last_tested_at->format('Y-m-d H:i:s'));
-        $this->assertEquals('working', $account->status);
+        $this->assertSame(AccountStatusEnum::HEALTHY, $account->status);
     }
 
     public function test_account_factory_states(): void
@@ -290,11 +291,11 @@ public function test_account_factory_states(): void
 
         $testedAccount = PlatformAccount::factory()->tested()->create();
         $this->assertNotNull($testedAccount->last_tested_at);
-        $this->assertEquals('working', $testedAccount->status);
+        $this->assertSame(AccountStatusEnum::HEALTHY, $testedAccount->status);
 
         $failedAccount = PlatformAccount::factory()->failed()->create();
         $this->assertNotNull($failedAccount->last_tested_at);
-        $this->assertEquals('failed', $failedAccount->status);
+        $this->assertSame(AccountStatusEnum::UNHEALTHY, $failedAccount->status);
     }
 
     public function test_account_update(): void