urlService->host($url); $path = parse_url($url, PHP_URL_PATH) ?? '/'; $body = Cache::remember( "crawler:robots:{$host}", config('crawler.robots_cache_ttl_seconds'), function () use ($host) { try { $response = Http::get("https://{$host}/robots.txt"); return $response->successful() ? $response->body() : ''; } catch (ConnectionException) { return ''; } } ); return (new RobotsTxt($body))->allows($path, $userAgent); } public function crawlDelayFor(string $host, string $userAgent): ?int { $body = Cache::remember( "crawler:robots:{$host}", config('crawler.robots_cache_ttl_seconds'), function () use ($host) { try { $response = Http::get("https://{$host}/robots.txt"); return $response->successful() ? $response->body() : ''; } catch (ConnectionException) { return ''; } } ); $delay = (new RobotsTxt($body))->crawlDelay($userAgent); return $delay !== null ? (int) $delay : null; } }