11 - Add PolitenessService and crawler delay config
This commit is contained in:
parent
69aa5d9d3e
commit
7171348370
4 changed files with 46 additions and 0 deletions
|
|
@ -61,3 +61,5 @@ AWS_BUCKET=
|
||||||
AWS_USE_PATH_STYLE_ENDPOINT=false
|
AWS_USE_PATH_STYLE_ENDPOINT=false
|
||||||
|
|
||||||
VITE_APP_NAME="${APP_NAME}"
|
VITE_APP_NAME="${APP_NAME}"
|
||||||
|
|
||||||
|
CRAWLER_MIN_DOMAIN_DELAY_SECONDS=10
|
||||||
|
|
|
||||||
19
app/Services/PolitenessService.php
Normal file
19
app/Services/PolitenessService.php
Normal file
|
|
@ -0,0 +1,19 @@
|
||||||
|
<?php
|
||||||
|
|
||||||
|
declare(strict_types=1);
|
||||||
|
|
||||||
|
namespace App\Services;
|
||||||
|
|
||||||
|
class PolitenessService
|
||||||
|
{
|
||||||
|
public function minDelayFor(string $domain): int
|
||||||
|
{
|
||||||
|
$configValue = config('crawler.min_domain_delay_seconds');
|
||||||
|
|
||||||
|
if ($configValue !== null) {
|
||||||
|
return $configValue;
|
||||||
|
}
|
||||||
|
|
||||||
|
return 10;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -41,4 +41,6 @@
|
||||||
*/
|
*/
|
||||||
|
|
||||||
'user_agent' => env('CRAWLER_USER_AGENT', 'TroveBot/0.1 (+https://trove.lvl0.xyz/bot)'),
|
'user_agent' => env('CRAWLER_USER_AGENT', 'TroveBot/0.1 (+https://trove.lvl0.xyz/bot)'),
|
||||||
|
|
||||||
|
'min_domain_delay_seconds' => env('CRAWLER_MIN_DOMAIN_DELAY_SECONDS', 10),
|
||||||
];
|
];
|
||||||
|
|
|
||||||
23
tests/Unit/Services/PolitenessServiceTest.php
Normal file
23
tests/Unit/Services/PolitenessServiceTest.php
Normal file
|
|
@ -0,0 +1,23 @@
|
||||||
|
<?php
|
||||||
|
|
||||||
|
declare(strict_types=1);
|
||||||
|
|
||||||
|
namespace Tests\Unit\Services;
|
||||||
|
|
||||||
|
use App\Services\PolitenessService;
|
||||||
|
use Tests\TestCase;
|
||||||
|
|
||||||
|
class PolitenessServiceTest extends TestCase
|
||||||
|
{
|
||||||
|
public function test_min_delay_for_returns_config_default(): void
|
||||||
|
{
|
||||||
|
$this->assertSame(10, (new PolitenessService)->minDelayFor('example.com'));
|
||||||
|
}
|
||||||
|
|
||||||
|
public function test_min_delay_for_respects_config_override(): void
|
||||||
|
{
|
||||||
|
config()->set('crawler.min_domain_delay_seconds', 30);
|
||||||
|
|
||||||
|
$this->assertSame(30, (new PolitenessService)->minDelayFor('example.com'));
|
||||||
|
}
|
||||||
|
}
|
||||||
Loading…
Reference in a new issue