95 - Add a platform credential health check

This commit is contained in:
myrmidex 2026-08-11 00:44:26 +02:00
parent 034d0f7eac
commit c40e167ab5
11 changed files with 354 additions and 13 deletions

View file

@ -2,6 +2,7 @@
namespace App\Actions;
use App\Enums\AccountStatusEnum;
use App\Exceptions\PlatformAuthException;
use App\Models\PlatformAccount;
use App\Models\PlatformInstance;
@ -46,7 +47,7 @@ public function execute(string $instanceDomain, string $username, string $passwo
'api_token' => $authResponse['jwt'] ?? null,
],
'is_active' => true,
'status' => 'active',
'status' => AccountStatusEnum::HEALTHY,
]);
});
}

View file

@ -0,0 +1,19 @@
<?php
namespace App\Enums;
enum AccountStatusEnum: string
{
case UNTESTED = 'untested';
case HEALTHY = 'healthy';
case UNHEALTHY = 'unhealthy';
public function label(): string
{
return match ($this) {
self::UNTESTED => 'Untested',
self::HEALTHY => 'Healthy',
self::UNHEALTHY => 'Unhealthy',
};
}
}

View file

@ -0,0 +1,79 @@
<?php
namespace App\Jobs;
use App\Enums\NotificationSeverityEnum;
use App\Enums\NotificationTypeEnum;
use App\Models\Notification;
use App\Models\PlatformAccount;
use App\Modules\Lemmy\Services\LemmyApiService;
use App\Services\Notification\NotificationService;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Queue\Queueable;
use Throwable;
class CheckPlatformCredentialsJob implements ShouldQueue
{
use Queueable;
public function handle(NotificationService $notificationService): void
{
$accounts = PlatformAccount::where('is_active', true)->get();
foreach ($accounts as $account) {
$this->check($account, $notificationService);
}
}
private function check(PlatformAccount $account, NotificationService $notificationService): void
{
if ($this->canLogIn($account)) {
$account->recordCredentialCheckPassed();
return;
}
$wasUnhealthy = $account->isUnhealthy();
$account->recordCredentialCheckFailed();
if (! $wasUnhealthy && $account->refresh()->isUnhealthy()) {
$this->notify($account, $notificationService);
}
}
private function canLogIn(PlatformAccount $account): bool
{
try {
return $this->makeApiService($account)->login($account->username, $account->password) !== null;
} catch (Throwable) {
return false;
}
}
protected function makeApiService(PlatformAccount $account): LemmyApiService
{
return new LemmyApiService($account->instance_url);
}
private function notify(PlatformAccount $account, NotificationService $notificationService): void
{
$alreadyNotified = Notification::query()
->where('type', NotificationTypeEnum::CREDENTIAL_EXPIRED)
->where('notifiable_type', $account->getMorphClass())
->where('notifiable_id', $account->getKey())
->unread()
->exists();
if ($alreadyNotified) {
return;
}
$notificationService->send(
type: NotificationTypeEnum::CREDENTIAL_EXPIRED,
severity: NotificationSeverityEnum::ERROR,
title: "Credentials failed for {$account->username}",
message: "Could not log in to {$account->instance_url} after ".PlatformAccount::FAILURES_BEFORE_UNHEALTHY.' attempts. Publishing to this account will fail until it is fixed.',
notifiable: $account,
);
}
}

View file

@ -2,6 +2,7 @@
namespace App\Models;
use App\Enums\AccountStatusEnum;
use App\Enums\PlatformEnum;
use Database\Factories\PlatformAccountFactory;
use Illuminate\Database\Eloquent\Casts\Attribute;
@ -21,7 +22,8 @@
* @property array<string, mixed> $settings
* @property bool $is_active
* @property Carbon|null $last_tested_at
* @property string $status
* @property AccountStatusEnum $status
* @property int $consecutive_failures
* @property Carbon $created_at
* @property Carbon $updated_at
* @property Collection<int, PlatformChannel> $activeChannels
@ -44,10 +46,12 @@ class PlatformAccount extends Model
'is_active',
'last_tested_at',
'status',
'consecutive_failures',
];
protected $casts = [
'platform' => PlatformEnum::class,
'status' => AccountStatusEnum::class,
'settings' => 'array',
'is_active' => 'boolean',
'last_tested_at' => 'datetime',
@ -136,4 +140,33 @@ public function activeChannels(): BelongsToMany
->wherePivot('is_active', true)
->orderByPivot('priority', 'desc');
}
public const FAILURES_BEFORE_UNHEALTHY = 3;
public function recordCredentialCheckPassed(): void
{
$this->update([
'status' => AccountStatusEnum::HEALTHY,
'consecutive_failures' => 0,
'last_tested_at' => now(),
]);
}
public function recordCredentialCheckFailed(): void
{
$failures = $this->consecutive_failures + 1;
$this->update([
'status' => $failures >= self::FAILURES_BEFORE_UNHEALTHY
? AccountStatusEnum::UNHEALTHY
: $this->status,
'consecutive_failures' => $failures,
'last_tested_at' => now(),
]);
}
public function isUnhealthy(): bool
{
return $this->status === AccountStatusEnum::UNHEALTHY;
}
}

View file

@ -2,6 +2,7 @@
namespace Database\Factories;
use App\Enums\AccountStatusEnum;
use App\Enums\PlatformEnum;
use App\Models\PlatformAccount;
use Illuminate\Database\Eloquent\Factories\Factory;
@ -23,7 +24,7 @@ public function definition(): array
'settings' => [],
'is_active' => true,
'last_tested_at' => null,
'status' => 'untested',
'status' => AccountStatusEnum::UNTESTED,
];
}
@ -38,7 +39,7 @@ public function tested(): static
{
return $this->state(fn (array $attributes) => [
'last_tested_at' => now()->subHours(2),
'status' => 'working',
'status' => AccountStatusEnum::HEALTHY,
]);
}
@ -46,7 +47,8 @@ public function failed(): static
{
return $this->state(fn (array $attributes) => [
'last_tested_at' => now()->subHours(2),
'status' => 'failed',
'status' => AccountStatusEnum::UNHEALTHY,
'consecutive_failures' => PlatformAccount::FAILURES_BEFORE_UNHEALTHY,
]);
}
}

View file

@ -0,0 +1,28 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
public function up(): void
{
Schema::table('platform_accounts', function (Blueprint $table) {
$table->unsignedTinyInteger('consecutive_failures')->default(0)->after('status');
});
DB::table('platform_accounts')->where('status', 'active')->update(['status' => 'healthy']);
}
public function down(): void
{
// Not a true inverse: accounts the health check marked healthy also become 'active'.
DB::table('platform_accounts')->where('status', 'healthy')->update(['status' => 'active']);
Schema::table('platform_accounts', function (Blueprint $table) {
$table->dropColumn('consecutive_failures');
});
}
};

View file

@ -69,7 +69,14 @@ class="text-sm text-blue-600 hover:text-blue-800 dark:text-blue-400"
<div class="space-y-1">
@foreach ($channel->platformAccounts->take(3) as $account)
<div class="flex items-center justify-between text-sm">
<span class="flex items-center gap-x-1.5">
<span class="text-gray-600 dark:text-gray-300">{{ $account->username }}</span>
@if ($account->isUnhealthy())
<span class="inline-flex items-center px-1.5 py-0.5 rounded-full text-xs font-medium bg-red-100 text-red-800 dark:bg-red-900/30 dark:text-red-300" title="Login failed on the last {{ App\Models\PlatformAccount::FAILURES_BEFORE_UNHEALTHY }} checks">
Credentials failing
</span>
@endif
</span>
<button
wire:click="detachAccount({{ $channel->id }}, {{ $account->id }})"
class="text-red-500 hover:text-red-700"

View file

@ -2,6 +2,7 @@
use App\Jobs\ArticleDiscoveryJob;
use App\Jobs\CheckFeedStalenessJob;
use App\Jobs\CheckPlatformCredentialsJob;
use App\Jobs\CleanupActivityLogsJob;
use App\Jobs\CleanupArticlesJob;
use App\Jobs\PublishNextArticleJob;
@ -30,6 +31,12 @@
->withoutOverlapping()
->onOneServer();
Schedule::job(new CheckPlatformCredentialsJob)
->daily()
->name('check-platform-credentials')
->withoutOverlapping()
->onOneServer();
Schedule::job(new CleanupArticlesJob)
->daily()
->name('cleanup-old-articles')

View file

@ -0,0 +1,163 @@
<?php
namespace Tests\Feature\Jobs;
use App\Enums\AccountStatusEnum;
use App\Enums\NotificationTypeEnum;
use App\Jobs\CheckPlatformCredentialsJob;
use App\Models\Notification;
use App\Models\PlatformAccount;
use App\Modules\Lemmy\Services\LemmyApiService;
use App\Services\Notification\NotificationService;
use Illuminate\Foundation\Testing\RefreshDatabase;
use Mockery;
use Tests\TestCase;
class CheckPlatformCredentialsJobTest extends TestCase
{
use RefreshDatabase;
private function runJob(bool $loginSucceeds): void
{
$api = Mockery::mock(LemmyApiService::class);
$api->shouldReceive('login')->andReturn($loginSucceeds ? 'a-token' : null);
$job = new class($api) extends CheckPlatformCredentialsJob
{
public function __construct(private LemmyApiService $api) {}
protected function makeApiService(PlatformAccount $account): LemmyApiService
{
return $this->api;
}
};
$job->handle(app(NotificationService::class));
}
/**
* @param array<string, mixed> $attributes
*/
private function account(array $attributes = []): PlatformAccount
{
return PlatformAccount::factory()->create(array_merge(['is_active' => true], $attributes));
}
public function test_a_successful_login_marks_the_account_healthy(): void
{
$account = $this->account(['status' => AccountStatusEnum::UNTESTED]);
$this->runJob(true);
$account->refresh();
$this->assertSame(AccountStatusEnum::HEALTHY, $account->status);
$this->assertSame(0, $account->consecutive_failures);
$this->assertNotNull($account->last_tested_at);
}
public function test_a_successful_login_clears_earlier_failures(): void
{
$account = $this->account(['status' => AccountStatusEnum::UNHEALTHY, 'consecutive_failures' => 2]);
$this->runJob(true);
$account->refresh();
$this->assertSame(AccountStatusEnum::HEALTHY, $account->status);
$this->assertSame(0, $account->consecutive_failures);
}
public function test_a_single_failure_does_not_mark_the_account_unhealthy(): void
{
$account = $this->account(['status' => AccountStatusEnum::HEALTHY]);
$this->runJob(false);
$account->refresh();
$this->assertSame(AccountStatusEnum::HEALTHY, $account->status);
$this->assertSame(1, $account->consecutive_failures);
}
public function test_the_threshold_failure_marks_the_account_unhealthy(): void
{
$account = $this->account([
'status' => AccountStatusEnum::HEALTHY,
'consecutive_failures' => PlatformAccount::FAILURES_BEFORE_UNHEALTHY - 1,
]);
$this->runJob(false);
$this->assertSame(AccountStatusEnum::UNHEALTHY, $account->refresh()->status);
}
public function test_a_single_failure_does_not_notify(): void
{
$this->account(['status' => AccountStatusEnum::HEALTHY]);
$this->runJob(false);
$this->assertDatabaseCount('notifications', 0);
}
public function test_becoming_unhealthy_notifies(): void
{
$account = $this->account([
'status' => AccountStatusEnum::HEALTHY,
'consecutive_failures' => PlatformAccount::FAILURES_BEFORE_UNHEALTHY - 1,
'username' => 'newsbot',
]);
$this->runJob(false);
$this->assertDatabaseHas('notifications', [
'type' => NotificationTypeEnum::CREDENTIAL_EXPIRED->value,
'notifiable_type' => $account->getMorphClass(),
'notifiable_id' => $account->id,
]);
$this->assertStringContainsString('newsbot', Notification::first()->title);
}
public function test_an_account_already_unhealthy_does_not_notify_again(): void
{
$this->account([
'status' => AccountStatusEnum::UNHEALTHY,
'consecutive_failures' => PlatformAccount::FAILURES_BEFORE_UNHEALTHY,
]);
$this->runJob(false);
$this->assertDatabaseCount('notifications', 0);
}
public function test_inactive_accounts_are_not_checked(): void
{
$account = $this->account(['is_active' => false, 'status' => AccountStatusEnum::UNTESTED]);
$this->runJob(false);
$account->refresh();
$this->assertSame(AccountStatusEnum::UNTESTED, $account->status);
$this->assertNull($account->last_tested_at);
}
public function test_an_exception_during_login_counts_as_a_failure(): void
{
$account = $this->account(['status' => AccountStatusEnum::HEALTHY]);
$api = Mockery::mock(LemmyApiService::class);
$api->shouldReceive('login')->andThrow(new \RuntimeException('instance unreachable'));
$job = new class($api) extends CheckPlatformCredentialsJob
{
public function __construct(private LemmyApiService $api) {}
protected function makeApiService(PlatformAccount $account): LemmyApiService
{
return $this->api;
}
};
$job->handle(app(NotificationService::class));
$this->assertSame(1, $account->refresh()->consecutive_failures);
}
}

View file

@ -3,6 +3,7 @@
namespace Tests\Unit\Actions;
use App\Actions\CreatePlatformAccountAction;
use App\Enums\AccountStatusEnum;
use App\Enums\PlatformEnum;
use App\Exceptions\PlatformAuthException;
use App\Models\PlatformAccount;
@ -54,7 +55,7 @@ public function test_creates_platform_account_with_new_instance(): void
$this->assertEquals('https://lemmy.world', $account->instance_url);
$this->assertEquals('lemmy', $account->platform->value);
$this->assertTrue($account->is_active);
$this->assertEquals('active', $account->status);
$this->assertSame(AccountStatusEnum::HEALTHY, $account->status);
$this->assertEquals(42, $account->settings['person_id']);
$this->assertEquals('Test User', $account->settings['display_name']);
$this->assertEquals('A test bio', $account->settings['description']);

View file

@ -2,6 +2,7 @@
namespace Tests\Unit\Models;
use App\Enums\AccountStatusEnum;
use App\Enums\PlatformEnum;
use App\Models\PlatformAccount;
use App\Models\PlatformChannel;
@ -15,7 +16,7 @@ class PlatformAccountTest extends TestCase
public function test_fillable_fields(): void
{
$fillableFields = ['platform', 'instance_url', 'username', 'password', 'settings', 'is_active', 'last_tested_at', 'status'];
$fillableFields = ['platform', 'instance_url', 'username', 'password', 'settings', 'is_active', 'last_tested_at', 'status', 'consecutive_failures'];
$account = new PlatformAccount;
$this->assertEquals($fillableFields, $account->getFillable());
@ -253,7 +254,7 @@ public function test_account_creation_with_factory(): void
$this->assertEquals('test-password', $account->password);
$this->assertIsBool($account->is_active);
$this->assertTrue($account->is_active);
$this->assertEquals('untested', $account->status);
$this->assertSame(AccountStatusEnum::UNTESTED, $account->status);
$this->assertIsArray($account->settings);
}
@ -270,7 +271,7 @@ public function test_account_creation_with_explicit_values(): void
'settings' => $settings,
'is_active' => false,
'last_tested_at' => $timestamp,
'status' => 'working',
'status' => AccountStatusEnum::HEALTHY,
]);
$this->assertEquals(PlatformEnum::LEMMY, $account->platform);
@ -280,7 +281,7 @@ public function test_account_creation_with_explicit_values(): void
$this->assertEquals($settings, $account->settings);
$this->assertFalse($account->is_active);
$this->assertEquals($timestamp->format('Y-m-d H:i:s'), $account->last_tested_at->format('Y-m-d H:i:s'));
$this->assertEquals('working', $account->status);
$this->assertSame(AccountStatusEnum::HEALTHY, $account->status);
}
public function test_account_factory_states(): void
@ -290,11 +291,11 @@ public function test_account_factory_states(): void
$testedAccount = PlatformAccount::factory()->tested()->create();
$this->assertNotNull($testedAccount->last_tested_at);
$this->assertEquals('working', $testedAccount->status);
$this->assertSame(AccountStatusEnum::HEALTHY, $testedAccount->status);
$failedAccount = PlatformAccount::factory()->failed()->create();
$this->assertNotNull($failedAccount->last_tested_at);
$this->assertEquals('failed', $failedAccount->status);
$this->assertSame(AccountStatusEnum::UNHEALTHY, $failedAccount->status);
}
public function test_account_update(): void