release/v1.3.5 #122

Merged
myrmidex merged 6 commits from release/v1.3.5 into main 2026-08-02 09:18:59 +02:00
20 changed files with 436 additions and 18 deletions

1
.gitignore vendored
View file

@ -24,3 +24,4 @@ yarn-error.log
/coverage-report*
/coverage.xml
/.php-cs-fixer.dist.php
/.php-cs-fixer.cache

View file

@ -110,12 +110,12 @@ public static function extractThumbnail(string $html): ?string
{
// Try OpenGraph image first
if (preg_match('/<meta property="og:image" content="([^"]+)"/i', $html, $matches)) {
return $matches[1];
return html_entity_decode($matches[1], ENT_QUOTES, 'UTF-8');
}
// Try first image in article content
if (preg_match('/<img[^>]+src="([^"]+)"/i', $html, $matches)) {
return $matches[1];
return html_entity_decode($matches[1], ENT_QUOTES, 'UTF-8');
}
return null;

View file

@ -0,0 +1,28 @@
<?php
namespace App\Services\Parsers;
class BelgaHomepageParser
{
private const ARTICLE_URL_TEMPLATE = 'https://www.belganewsagency.eu/press-releases/%s/';
/**
* @return array<int, string>
*/
public static function extractArticleUrls(string $json): array
{
$decoded = json_decode($json, true);
if (! is_array($decoded) || ! isset($decoded['data']) || ! is_array($decoded['data'])) {
return [];
}
return collect($decoded['data'])
->pluck('id')
->filter(fn ($id) => is_int($id) || (is_string($id) && ctype_digit($id)))
->map(fn ($id) => sprintf(self::ARTICLE_URL_TEMPLATE, $id))
->unique()
->values()
->toArray();
}
}

View file

@ -0,0 +1,39 @@
<?php
namespace App\Services\Parsers;
use App\Contracts\HomepageParserInterface;
class BelgaHomepageParserAdapter implements HomepageParserInterface
{
/**
* Must stay in sync with config('feed.providers.belga.languages.en.url').
* Nothing calls getHomepageUrl() at runtime the fetcher uses $feed->url,
* seeded from that config value so drift here would go unnoticed.
*/
private const API_URL = 'https://capi.belga.press/belgapress/api/public/pressreleases?offset=1&count=6&search=&start=&end=&newsroomId=70&language=EN';
public function __construct(
private string $language = 'en',
) {}
public function canParse(string $url): bool
{
return str_contains($url, 'belganewsagency.eu') || str_contains($url, 'capi.belga.press');
}
public function extractArticleUrls(string $html): array
{
return BelgaHomepageParser::extractArticleUrls($html);
}
public function getHomepageUrl(): string
{
return str_replace('language=EN', 'language='.strtoupper($this->language), self::API_URL);
}
public function getSourceName(): string
{
return 'Belga News Agency';
}
}

View file

@ -68,12 +68,12 @@ public static function extractThumbnail(string $html): ?string
{
// Try OpenGraph image first
if (preg_match('/<meta property="og:image" content="([^"]+)"/i', $html, $matches)) {
return $matches[1];
return html_entity_decode($matches[1], ENT_QUOTES, 'UTF-8');
}
// Try first image in content
if (preg_match('/<img[^>]+src="([^"]+)"/i', $html, $matches)) {
return $matches[1];
return html_entity_decode($matches[1], ENT_QUOTES, 'UTF-8');
}
return null;

View file

@ -67,11 +67,11 @@ public static function extractFullArticle(string $html): ?string
public static function extractThumbnail(string $html): ?string
{
if (preg_match('/<meta property="og:image" content="([^"]+)"/i', $html, $matches)) {
return $matches[1];
return html_entity_decode($matches[1], ENT_QUOTES, 'UTF-8');
}
if (preg_match('/<img[^>]+src="([^"]+)"/i', $html, $matches)) {
return $matches[1];
return html_entity_decode($matches[1], ENT_QUOTES, 'UTF-8');
}
return null;

View file

@ -13,7 +13,7 @@
|
*/
'name' => env('APP_NAME', 'Laravel'),
'name' => 'Fedi Feed Router',
/*
|--------------------------------------------------------------------------

View file

@ -2,6 +2,7 @@
use App\Services\Parsers\BelgaArticlePageParser;
use App\Services\Parsers\BelgaArticleParser;
use App\Services\Parsers\BelgaHomepageParserAdapter;
use App\Services\Parsers\GuardianArticlePageParser;
use App\Services\Parsers\GuardianArticleParser;
use App\Services\Parsers\VrtArticlePageParser;
@ -41,12 +42,13 @@
'code' => 'belga',
'name' => 'Belga News Agency',
'description' => 'Belgian national news agency',
'type' => 'rss',
'type' => 'website',
'is_active' => true,
'languages' => [
'en' => ['url' => 'https://www.belganewsagency.eu/feed'],
'en' => ['url' => 'https://capi.belga.press/belgapress/api/public/pressreleases?offset=1&count=6&search=&start=&end=&newsroomId=70&language=EN'],
],
'parsers' => [
'homepage' => BelgaHomepageParserAdapter::class,
'article' => BelgaArticleParser::class,
'article_page' => BelgaArticlePageParser::class,
],

3
pint.json Normal file
View file

@ -0,0 +1,3 @@
{
"preset": "laravel"
}

View file

@ -5,7 +5,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1">
<meta name="csrf-token" content="{{ csrf_token() }}">
<title>{{ config('app.name', 'FFR') }}</title>
<title>{{ config('app.name') }}</title>
<!-- Fonts -->
<link rel="preconnect" href="https://fonts.bunny.net">

View file

@ -5,7 +5,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1">
<meta name="csrf-token" content="{{ csrf_token() }}">
<title>{{ config('app.name', 'FFR') }}</title>
<title>{{ config('app.name') }}</title>
<!-- Fonts -->
<link rel="preconnect" href="https://fonts.bunny.net">

View file

@ -5,7 +5,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1">
<meta name="csrf-token" content="{{ csrf_token() }}">
<title>{{ config('app.name', 'FFR') }} - Setup</title>
<title>{{ config('app.name') }} - Setup</title>
<!-- Fonts -->
<link rel="preconnect" href="https://fonts.bunny.net">

View file

@ -92,22 +92,24 @@ public function test_store_creates_belga_feed_successfully(): void
$response = $this->postJson('/api/v1/feeds', $feedData);
$expectedUrl = config('feed.providers.belga.languages.en.url');
$response->assertStatus(201)
->assertJson([
'success' => true,
'message' => 'Feed created successfully!',
'data' => [
'name' => 'Belga Test Feed',
'url' => 'https://www.belganewsagency.eu/feed',
'type' => 'rss',
'url' => $expectedUrl,
'type' => 'website',
'is_active' => true,
],
]);
$this->assertDatabaseHas('feeds', [
'name' => 'Belga Test Feed',
'url' => 'https://www.belganewsagency.eu/feed',
'type' => 'rss',
'url' => $expectedUrl,
'type' => 'website',
]);
}

File diff suppressed because one or more lines are too long

View file

@ -42,8 +42,9 @@ public function test_creates_belga_feed_with_correct_url(): void
$feed = $this->action->execute('Belga News', 'belga', $language->id);
$this->assertEquals('https://www.belganewsagency.eu/feed', $feed->url);
$this->assertEquals('rss', $feed->type);
$this->assertStringStartsWith('https://capi.belga.press/belgapress/api/public/pressreleases?', $feed->url);
$this->assertStringContainsString('newsroomId=70', $feed->url);
$this->assertEquals('website', $feed->type);
$this->assertEquals('belga', $feed->provider);
$this->assertNull($feed->description);
}

View file

@ -0,0 +1,100 @@
<?php
namespace Tests\Unit\Services;
use App\Models\Feed;
use App\Models\Language;
use Illuminate\Foundation\Testing\RefreshDatabase;
use Illuminate\Support\Facades\Http;
use Mockery;
use Tests\TestCase;
use Tests\Traits\CreatesArticleFetcher;
/**
* Belga discovery runs through the website path against a JSON API rather than
* a rendered homepage. This lives in its own file because ArticleFetcherTest
* registers a catch-all Http::fake in setUp() that a per-test fake cannot
* override.
*/
class ArticleFetcherBelgaTest extends TestCase
{
use CreatesArticleFetcher, RefreshDatabase;
private function apiResponse(): string
{
$contents = file_get_contents(__DIR__.'/../../Fixtures/belga-pressreleases.json');
$this->assertNotFalse($contents, 'Belga fixture could not be read.');
return $contents;
}
private function belgaFeed(): Feed
{
$language = Language::factory()->create(['short_code' => 'en']);
return Feed::factory()->create([
'type' => 'website',
'provider' => 'belga',
'language_id' => $language->id,
'url' => config('feed.providers.belga.languages.en.url'),
]);
}
public function test_creates_articles_from_belga_api_response(): void
{
Http::fake(['*' => Http::response($this->apiResponse(), 200)]);
$result = $this->createArticleFetcher()->getArticlesFromFeed($this->belgaFeed());
$this->assertCount(6, $result);
$this->assertDatabaseHas('articles', [
'url' => 'https://www.belganewsagency.eu/press-releases/35285/',
]);
$this->assertDatabaseHas('articles', [
'url' => 'https://www.belganewsagency.eu/press-releases/35269/',
]);
}
public function test_associates_created_articles_with_the_feed(): void
{
Http::fake(['*' => Http::response($this->apiResponse(), 200)]);
$feed = $this->belgaFeed();
$this->createArticleFetcher()->getArticlesFromFeed($feed);
$this->assertDatabaseHas('articles', [
'url' => 'https://www.belganewsagency.eu/press-releases/35285/',
'feed_id' => $feed->id,
]);
}
public function test_returns_empty_collection_when_api_returns_no_articles(): void
{
// A well-formed envelope with nothing in it is legitimate (quiet day),
// and must be a no-op rather than an error.
Http::fake(['*' => Http::response('{"data":[],"_meta":{"total":0}}', 200)]);
$result = $this->createArticleFetcher()->getArticlesFromFeed($this->belgaFeed());
$this->assertEmpty($result);
$this->assertDatabaseCount('articles', 0);
}
public function test_returns_empty_collection_when_api_returns_an_error_page(): void
{
// The failure mode that caused #115: a 404 HTML body reaching the parser.
Http::fake(['*' => Http::response('<html><body>404 Not Found</body></html>', 200)]);
$result = $this->createArticleFetcher()->getArticlesFromFeed($this->belgaFeed());
$this->assertEmpty($result);
}
protected function tearDown(): void
{
Mockery::close();
parent::tearDown();
}
}

View file

@ -242,6 +242,36 @@ public function test_extract_thumbnail_prefers_og_image(): void
$this->assertEquals('https://example.com/og-image.jpg', $thumbnail);
}
public function test_extract_thumbnail_decodes_html_entities_in_og_image(): void
{
// Real Belga CDN URLs carry query params, so the &amp; in the raw HTML
// must be decoded or the URL Lemmy receives is malformed. See #117.
$html = '<html><head><meta property="og:image" content="https://cdn.example.com/img:12345:full?v=6a6c828f&amp;m=cdodnjha"/></head></html>';
$thumbnail = BelgaArticlePageParser::extractThumbnail($html);
$this->assertEquals('https://cdn.example.com/img:12345:full?v=6a6c828f&m=cdodnjha', $thumbnail);
$this->assertStringNotContainsString('&amp;', (string) $thumbnail);
}
public function test_extract_thumbnail_decodes_html_entities_in_img_tag(): void
{
$html = '<html><body><img src="https://cdn.example.com/pic.png?a=1&amp;b=2" alt="test"/></body></html>';
$thumbnail = BelgaArticlePageParser::extractThumbnail($html);
$this->assertEquals('https://cdn.example.com/pic.png?a=1&b=2', $thumbnail);
}
public function test_extract_data_returns_decoded_thumbnail(): void
{
$html = '<html><head><meta property="og:image" content="https://cdn.example.com/x.jpg?v=1&amp;m=2"/></head></html>';
$data = BelgaArticlePageParser::extractData($html);
$this->assertEquals('https://cdn.example.com/x.jpg?v=1&m=2', $data['thumbnail']);
}
public function test_extract_thumbnail_returns_null_when_not_found(): void
{
$html = '<html><body><div>No images here</div></body></html>';

View file

@ -0,0 +1,116 @@
<?php
namespace Tests\Unit\Services\Parsers;
use App\Services\Parsers\BelgaHomepageParser;
use App\Services\Parsers\BelgaHomepageParserAdapter;
use PHPUnit\Framework\TestCase;
class BelgaHomepageParserTest extends TestCase
{
private function fixture(): string
{
$contents = file_get_contents(__DIR__.'/../../../Fixtures/belga-pressreleases.json');
$this->assertNotFalse($contents, 'Belga fixture could not be read.');
return $contents;
}
/**
* @param array<int, array<string, mixed>> $data
*/
private function jsonWith(array $data): string
{
return (string) json_encode(['data' => $data]);
}
public function test_extracts_article_urls_from_api_json(): void
{
$urls = BelgaHomepageParser::extractArticleUrls($this->fixture());
$this->assertCount(6, $urls);
$this->assertSame('https://www.belganewsagency.eu/press-releases/35285/', $urls[0]);
$this->assertContains('https://www.belganewsagency.eu/press-releases/35269/', $urls);
}
public function test_deduplicates_urls(): void
{
$json = $this->jsonWith([
['id' => 100],
['id' => 100],
['id' => '100'],
['id' => 101],
]);
$urls = BelgaHomepageParser::extractArticleUrls($json);
$this->assertSame([
'https://www.belganewsagency.eu/press-releases/100/',
'https://www.belganewsagency.eu/press-releases/101/',
], $urls);
}
public function test_returns_empty_array_for_malformed_json(): void
{
$this->assertSame([], BelgaHomepageParser::extractArticleUrls('<html>404 not found</html>'));
$this->assertSame([], BelgaHomepageParser::extractArticleUrls('{"data": ['));
$this->assertSame([], BelgaHomepageParser::extractArticleUrls(''));
}
public function test_returns_empty_array_when_data_key_missing_or_empty(): void
{
$this->assertSame([], BelgaHomepageParser::extractArticleUrls('{}'));
$this->assertSame([], BelgaHomepageParser::extractArticleUrls('{"data": []}'));
$this->assertSame([], BelgaHomepageParser::extractArticleUrls('{"data": "not-an-array"}'));
}
public function test_skips_entries_without_usable_id(): void
{
$json = $this->jsonWith([
['id' => 200],
['id' => null],
['publishDate' => '2026-08-01T06:24:19'],
['id' => 'not-numeric'],
['id' => 201],
]);
$urls = BelgaHomepageParser::extractArticleUrls($json);
$this->assertSame([
'https://www.belganewsagency.eu/press-releases/200/',
'https://www.belganewsagency.eu/press-releases/201/',
], $urls);
}
public function test_adapter_delegates_and_reports_source(): void
{
$adapter = new BelgaHomepageParserAdapter('en');
$this->assertSame(
BelgaHomepageParser::extractArticleUrls($this->fixture()),
$adapter->extractArticleUrls($this->fixture())
);
$this->assertTrue($adapter->canParse('https://www.belganewsagency.eu/press-releases/35285/'));
$this->assertTrue($adapter->canParse('https://capi.belga.press/belgapress/api/public/pressreleases'));
$this->assertFalse($adapter->canParse('https://www.vrt.be/vrtnws/en/'));
$this->assertSame('Belga News Agency', $adapter->getSourceName());
}
public function test_adapter_uppercases_language_in_api_url(): void
{
$this->assertStringContainsString('language=EN', (new BelgaHomepageParserAdapter('en'))->getHomepageUrl());
$this->assertStringContainsString('language=NL', (new BelgaHomepageParserAdapter('nl'))->getHomepageUrl());
}
public function test_adapter_api_url_pins_load_bearing_query_params(): void
{
$url = (new BelgaHomepageParserAdapter('en'))->getHomepageUrl();
$this->assertStringStartsWith('https://capi.belga.press/belgapress/api/public/pressreleases?', $url);
$this->assertStringContainsString('newsroomId=70', $url);
$this->assertStringContainsString('offset=1', $url);
$this->assertStringContainsString('count=6', $url);
}
}

View file

@ -282,4 +282,24 @@ public function test_extract_full_article_with_realistic_guardian_html(): void
$this->assertStringContainsString("\n\n", $fullArticle);
$this->assertStringNotContainsString('<strong>', $fullArticle);
}
public function test_extract_thumbnail_decodes_html_entities(): void
{
// Entity-encoded ampersands make the URL unusable downstream. See #117.
$html = '<html><head><meta property="og:image" content="https://i.guim.co.uk/img/media/abc.jpg?width=1200&amp;quality=85"/></head></html>';
$thumbnail = GuardianArticlePageParser::extractThumbnail($html);
$this->assertEquals('https://i.guim.co.uk/img/media/abc.jpg?width=1200&quality=85', $thumbnail);
$this->assertStringNotContainsString('&amp;', (string) $thumbnail);
}
public function test_extract_thumbnail_decodes_html_entities_in_img_tag(): void
{
$html = '<html><body><img src="https://i.guim.co.uk/img/pic.png?a=1&amp;b=2" alt="test"/></body></html>';
$thumbnail = GuardianArticlePageParser::extractThumbnail($html);
$this->assertEquals('https://i.guim.co.uk/img/pic.png?a=1&b=2', $thumbnail);
}
}

View file

@ -0,0 +1,75 @@
<?php
namespace Tests\Unit\Services\Parsers;
use App\Services\Parsers\VrtArticlePageParser;
use Tests\TestCase;
class VrtArticlePageParserTest extends TestCase
{
public function test_extract_thumbnail_from_og_image(): void
{
$html = '<html><head><meta property="og:image" content="https://images.vrt.be/image.jpg"/></head></html>';
$this->assertEquals(
'https://images.vrt.be/image.jpg',
VrtArticlePageParser::extractThumbnail($html)
);
}
public function test_extract_thumbnail_falls_back_to_img_tag(): void
{
$html = '<html><body><img src="https://images.vrt.be/body.png" alt="test"/></body></html>';
$this->assertEquals(
'https://images.vrt.be/body.png',
VrtArticlePageParser::extractThumbnail($html)
);
}
public function test_extract_thumbnail_prefers_og_image(): void
{
$html = '<html><head><meta property="og:image" content="https://images.vrt.be/og.jpg"/></head>'
.'<body><img src="https://images.vrt.be/body.png" alt="test"/></body></html>';
$this->assertEquals(
'https://images.vrt.be/og.jpg',
VrtArticlePageParser::extractThumbnail($html)
);
}
public function test_extract_thumbnail_returns_null_when_not_found(): void
{
$this->assertNull(VrtArticlePageParser::extractThumbnail('<html><body>No images</body></html>'));
}
public function test_extract_thumbnail_decodes_html_entities(): void
{
// Entity-encoded ampersands make the URL unusable downstream. See #117.
$html = '<html><head><meta property="og:image" content="https://images.vrt.be/orig.jpg?crop=1&amp;w=1200"/></head></html>';
$thumbnail = VrtArticlePageParser::extractThumbnail($html);
$this->assertEquals('https://images.vrt.be/orig.jpg?crop=1&w=1200', $thumbnail);
$this->assertStringNotContainsString('&amp;', (string) $thumbnail);
}
public function test_extract_thumbnail_decodes_html_entities_in_img_tag(): void
{
$html = '<html><body><img src="https://images.vrt.be/pic.png?a=1&amp;b=2" alt="test"/></body></html>';
$this->assertEquals(
'https://images.vrt.be/pic.png?a=1&b=2',
VrtArticlePageParser::extractThumbnail($html)
);
}
public function test_extract_data_returns_decoded_thumbnail(): void
{
$html = '<html><head><meta property="og:image" content="https://images.vrt.be/x.jpg?v=1&amp;m=2"/></head></html>';
$data = VrtArticlePageParser::extractData($html);
$this->assertEquals('https://images.vrt.be/x.jpg?v=1&m=2', $data['thumbnail']);
}
}