createQuietly(['url' => 'https://example.com/article']); PageCrawl::factory()->page($page)->create(); Queue::assertPushed(ProcessCrawlJob::class); } public function test_dispatched_job_carries_the_correct_page_crawl(): void { Queue::fake(); $page = Page::factory()->createQuietly(['url' => 'https://example.com/article']); $crawl = PageCrawl::factory()->page($page)->create(); Queue::assertPushed( ProcessCrawlJob::class, fn (ProcessCrawlJob $job) => $job->pageCrawl->id === $crawl->id, ); } public function test_handle_writes_outcome_to_page_crawl_on_success(): void { Queue::fake(); $fetcher = Mockery::mock(FetchPageAction::class); $fetcher->shouldReceive('__invoke')->andReturn(new FetchResult( outcome: CrawlOutcomeEnum::Success, statusCode: 200, finalUrl: 'https://example.com/article', title: 'Hello', extractedText: 'hi', outboundLinks: collect(), wordCount: 1, errorMessage: null, )); $this->app->instance(FetchPageAction::class, $fetcher); $page = Page::factory()->createQuietly(['url' => 'https://example.com/article']); $crawl = PageCrawl::factory()->page($page)->createQuietly(); app(ProcessCrawlJob::class, ['pageCrawl' => $crawl]) ->handle(app(FetchPageAction::class), app(RegisterDiscoveredPageAction::class)); $fresh = $crawl->fresh(); $this->assertSame(CrawlOutcomeEnum::Success, $fresh->outcome); $this->assertNotNull($fresh->completed_at); $this->assertInstanceOf(Carbon::class, $fresh->completed_at); $this->assertSame(200, $fresh->status_code); $this->assertNull($fresh->error_message); } public function test_handle_updates_page_to_fetched_on_success(): void { Queue::fake(); $fetcher = Mockery::mock(FetchPageAction::class); $fetcher->shouldReceive('__invoke')->andReturn(new FetchResult( outcome: CrawlOutcomeEnum::Success, statusCode: 200, finalUrl: 'https://example.com/article', title: 'Hello', extractedText: 'hi', outboundLinks: collect(), wordCount: 1, errorMessage: null, )); $this->app->instance(FetchPageAction::class, $fetcher); $page = Page::factory()->createQuietly(['url' => 'https://example.com/article']); $crawl = PageCrawl::factory()->page($page)->createQuietly(); app(ProcessCrawlJob::class, ['pageCrawl' => $crawl]) ->handle(app(FetchPageAction::class), app(RegisterDiscoveredPageAction::class)); $fresh = $page->fresh(); $this->assertSame(PageStatusEnum::Fetched, $fresh->status); $this->assertNotNull($fresh->fetched_at); $this->assertInstanceOf(Carbon::class, $fresh->fetched_at); $this->assertSame('Hello', $fresh->title); } public function test_handle_updates_page_to_rejected_on_rejected_outcome(): void { Queue::fake(); $fetcher = Mockery::mock(FetchPageAction::class); $fetcher->shouldReceive('__invoke')->andReturn(new FetchResult( outcome: CrawlOutcomeEnum::Rejected, statusCode: 200, finalUrl: null, title: null, extractedText: null, outboundLinks: collect(), wordCount: null, errorMessage: 'Unsupported Content-Type: application/pdf', )); $this->app->instance(FetchPageAction::class, $fetcher); $page = Page::factory()->createQuietly(['url' => 'https://example.com/brochure.pdf']); $crawl = PageCrawl::factory()->page($page)->createQuietly(); app(ProcessCrawlJob::class, ['pageCrawl' => $crawl]) ->handle(app(FetchPageAction::class), app(RegisterDiscoveredPageAction::class)); $fresh = $page->fresh(); $this->assertSame(PageStatusEnum::Rejected, $fresh->status); $this->assertNull($fresh->fetched_at); } public function test_handle_updates_page_to_failed_on_blocked_4xx(): void { Queue::fake(); $fetcher = Mockery::mock(FetchPageAction::class); $fetcher->shouldReceive('__invoke')->andReturn(new FetchResult( outcome: CrawlOutcomeEnum::Blocked4xx, statusCode: 404, finalUrl: null, title: null, extractedText: null, outboundLinks: collect(), wordCount: null, errorMessage: 'HTTP 404', )); $this->app->instance(FetchPageAction::class, $fetcher); $page = Page::factory()->createQuietly(['url' => 'https://example.com/gone']); $crawl = PageCrawl::factory()->page($page)->createQuietly(); app(ProcessCrawlJob::class, ['pageCrawl' => $crawl]) ->handle(app(FetchPageAction::class), app(RegisterDiscoveredPageAction::class)); $fresh = $page->fresh(); $this->assertSame(PageStatusEnum::Failed, $fresh->status); $this->assertNotNull($fresh->failed_at); $this->assertInstanceOf(Carbon::class, $fresh->failed_at); } public function test_handle_updates_page_to_failed_on_timeout(): void { Queue::fake(); $fetcher = Mockery::mock(FetchPageAction::class); $fetcher->shouldReceive('__invoke')->andReturn(new FetchResult( outcome: CrawlOutcomeEnum::Timeout, statusCode: null, finalUrl: null, title: null, extractedText: null, outboundLinks: collect(), wordCount: null, errorMessage: 'Connection timed out after 10 seconds', )); $this->app->instance(FetchPageAction::class, $fetcher); $page = Page::factory()->createQuietly(['url' => 'https://example.com/slow']); $crawl = PageCrawl::factory()->page($page)->createQuietly(); app(ProcessCrawlJob::class, ['pageCrawl' => $crawl]) ->handle(app(FetchPageAction::class), app(RegisterDiscoveredPageAction::class)); $fresh = $page->fresh(); $this->assertSame(PageStatusEnum::Failed, $fresh->status); $this->assertNotNull($fresh->failed_at); $this->assertInstanceOf(Carbon::class, $fresh->failed_at); } public function test_handle_schedules_retry_on_transient_failure(): void { Queue::fake(); $fetcher = Mockery::mock(FetchPageAction::class); $fetcher->shouldReceive('__invoke')->andReturn(new FetchResult( outcome: CrawlOutcomeEnum::Failed, statusCode: null, finalUrl: null, title: null, extractedText: null, outboundLinks: collect(), wordCount: null, errorMessage: 'Connection refused', )); $this->app->instance(FetchPageAction::class, $fetcher); $page = Page::factory()->createQuietly(['url' => 'https://example.com/unstable']); $crawl = PageCrawl::factory()->page($page)->createQuietly(); app(ProcessCrawlJob::class, ['pageCrawl' => $crawl]) ->handle(app(FetchPageAction::class), app(RegisterDiscoveredPageAction::class)); // A second PageCrawl row (the retry) must have been inserted for the same page $this->assertSame(2, PageCrawl::where('page_id', $page->id)->count()); // The new row is pending — outcome IS NULL $retryRow = PageCrawl::where('page_id', $page->id) ->whereNull('outcome') ->first(); $this->assertNotNull($retryRow); // A delayed ProcessCrawlJob must have been pushed for the retry row Queue::assertPushed( ProcessCrawlJob::class, fn (ProcessCrawlJob $job) => $job->pageCrawl->page_id === $page->id && $job->pageCrawl->id === $retryRow->id, ); } public function test_handle_does_not_retry_after_three_attempts(): void { Queue::fake(); $fetcher = Mockery::mock(FetchPageAction::class); $fetcher->shouldReceive('__invoke')->andReturn(new FetchResult( outcome: CrawlOutcomeEnum::Failed, statusCode: null, finalUrl: null, title: null, extractedText: null, outboundLinks: collect(), wordCount: null, errorMessage: 'Connection refused', )); $this->app->instance(FetchPageAction::class, $fetcher); $page = Page::factory()->createQuietly(['url' => 'https://example.com/unreachable']); // 3 prior attempts already exist — this is the cap PageCrawl::factory()->page($page)->failed('Connection refused')->createQuietly(); PageCrawl::factory()->page($page)->failed('Connection refused')->createQuietly(); $thirdCrawl = PageCrawl::factory()->page($page)->createQuietly(); app(ProcessCrawlJob::class, ['pageCrawl' => $thirdCrawl]) ->handle(app(FetchPageAction::class), app(RegisterDiscoveredPageAction::class)); // No 4th row must appear — retry cap reached $this->assertSame(3, PageCrawl::where('page_id', $page->id)->count()); // No retry job dispatched Queue::assertNotPushed(ProcessCrawlJob::class); } public function test_handle_writes_failed_outcome_to_page_crawl(): void { Queue::fake(); $fetcher = Mockery::mock(FetchPageAction::class); $fetcher->shouldReceive('__invoke')->andReturn(new FetchResult( outcome: CrawlOutcomeEnum::Failed, statusCode: null, finalUrl: null, title: null, extractedText: null, outboundLinks: collect(), wordCount: null, errorMessage: 'boom', )); $this->app->instance(FetchPageAction::class, $fetcher); $page = Page::factory()->createQuietly(['url' => 'https://example.com/unstable']); $crawl = PageCrawl::factory()->page($page)->createQuietly(); app(ProcessCrawlJob::class, ['pageCrawl' => $crawl]) ->handle(app(FetchPageAction::class), app(RegisterDiscoveredPageAction::class)); $this->assertDatabaseHas('page_crawls', [ 'id' => $crawl->id, 'outcome' => CrawlOutcomeEnum::Failed->value, 'status_code' => null, 'error_message' => 'boom', ]); } public function test_handle_updates_page_to_failed_on_failed_outcome(): void { Queue::fake(); $fetcher = Mockery::mock(FetchPageAction::class); $fetcher->shouldReceive('__invoke')->andReturn(new FetchResult( outcome: CrawlOutcomeEnum::Failed, statusCode: null, finalUrl: null, title: null, extractedText: null, outboundLinks: collect(), wordCount: null, errorMessage: 'Connection refused', )); $this->app->instance(FetchPageAction::class, $fetcher); $page = Page::factory()->createQuietly(['url' => 'https://example.com/unreachable']); $crawl = PageCrawl::factory()->page($page)->createQuietly(); app(ProcessCrawlJob::class, ['pageCrawl' => $crawl]) ->handle(app(FetchPageAction::class), app(RegisterDiscoveredPageAction::class)); $this->assertSame(PageStatusEnum::Failed, $page->fresh()->status); } public function test_handle_updates_page_to_failed_on_blocked_5xx(): void { Queue::fake(); $fetcher = Mockery::mock(FetchPageAction::class); $fetcher->shouldReceive('__invoke')->andReturn(new FetchResult( outcome: CrawlOutcomeEnum::Blocked5xx, statusCode: 503, finalUrl: null, title: null, extractedText: null, outboundLinks: collect(), wordCount: null, errorMessage: 'HTTP 503', )); $this->app->instance(FetchPageAction::class, $fetcher); $page = Page::factory()->createQuietly(['url' => 'https://example.com/overloaded']); $crawl = PageCrawl::factory()->page($page)->createQuietly(); app(ProcessCrawlJob::class, ['pageCrawl' => $crawl]) ->handle(app(FetchPageAction::class), app(RegisterDiscoveredPageAction::class)); $this->assertSame(PageStatusEnum::Failed, $page->fresh()->status); } public function test_handle_updates_page_to_failed_on_blocked_robots(): void { Queue::fake(); $fetcher = Mockery::mock(FetchPageAction::class); $fetcher->shouldReceive('__invoke')->andReturn(new FetchResult( outcome: CrawlOutcomeEnum::BlockedRobots, statusCode: null, finalUrl: null, title: null, extractedText: null, outboundLinks: collect(), wordCount: null, errorMessage: 'Disallowed by robots.txt', )); $this->app->instance(FetchPageAction::class, $fetcher); $page = Page::factory()->createQuietly(['url' => 'https://example.com/private']); $crawl = PageCrawl::factory()->page($page)->createQuietly(); app(ProcessCrawlJob::class, ['pageCrawl' => $crawl]) ->handle(app(FetchPageAction::class), app(RegisterDiscoveredPageAction::class)); $this->assertSame(PageStatusEnum::Failed, $page->fresh()->status); } public function test_handle_does_not_register_outbound_links_on_failure(): void { Queue::fake(); $fetcher = Mockery::mock(FetchPageAction::class); $fetcher->shouldReceive('__invoke')->andReturn(new FetchResult( outcome: CrawlOutcomeEnum::Failed, statusCode: null, finalUrl: null, title: null, extractedText: null, outboundLinks: collect([ 'https://should-not-be-registered.com/page', ]), wordCount: null, errorMessage: 'Connection refused', )); $this->app->instance(FetchPageAction::class, $fetcher); $page = Page::factory()->createQuietly(['url' => 'https://example.com/broken']); $crawl = PageCrawl::factory()->page($page)->createQuietly(); app(ProcessCrawlJob::class, ['pageCrawl' => $crawl]) ->handle(app(FetchPageAction::class), app(RegisterDiscoveredPageAction::class)); $this->assertDatabaseMissing('pages', ['url' => 'https://should-not-be-registered.com/page']); $this->assertSame(1, Page::count()); } public function test_handle_registers_outbound_links_on_success(): void { Queue::fake(); $fetcher = Mockery::mock(FetchPageAction::class); $fetcher->shouldReceive('__invoke')->andReturn(new FetchResult( outcome: CrawlOutcomeEnum::Success, statusCode: 200, finalUrl: 'https://source.com/article', title: 'Source Article', extractedText: 'some text', outboundLinks: collect([ 'https://other.com/article-1', 'https://another.com/post-2', ]), wordCount: 2, errorMessage: null, )); $this->app->instance(FetchPageAction::class, $fetcher); $page = Page::factory()->createQuietly(['url' => 'https://source.com/article']); $crawl = PageCrawl::factory()->page($page)->createQuietly(); app(ProcessCrawlJob::class, ['pageCrawl' => $crawl]) ->handle(app(FetchPageAction::class), app(RegisterDiscoveredPageAction::class)); $this->assertDatabaseHas('pages', ['url' => 'https://other.com/article-1']); $this->assertDatabaseHas('pages', ['url' => 'https://another.com/post-2']); $this->assertSame(3, Page::count()); } }