From f141ab788953024c13f64cb839a7f459e3538a8a Mon Sep 17 00:00:00 2001
From: myrmidex <myrmidex@myrmidex.net>
Date: Sun, 29 Jun 2025 09:37:49 +0200
Subject: [PATCH] Add scheduler, fetch service

---
 .../Commands/FetchNewArticlesCommand.php      | 22 +++++++
 app/Models/Article.php                        | 27 ++++++++
 app/Services/Articles/ArticleFetcher.php      | 64 +++++++++++++++++++
 database/factories/ArticleFactory.php         | 23 +++++++
 ...025_06_29_072202_create_articles_table.php | 22 +++++++
 docker-compose.yml                            | 19 ++++++
 routes/console.php                            |  8 +--
 7 files changed, 180 insertions(+), 5 deletions(-)
 create mode 100644 app/Console/Commands/FetchNewArticlesCommand.php
 create mode 100644 app/Models/Article.php
 create mode 100644 app/Services/Articles/ArticleFetcher.php
 create mode 100644 database/factories/ArticleFactory.php
 create mode 100644 database/migrations/2025_06_29_072202_create_articles_table.php

diff --git a/app/Console/Commands/FetchNewArticlesCommand.php b/app/Console/Commands/FetchNewArticlesCommand.php
new file mode 100644
index 0000000..445a540
--- /dev/null
+++ b/app/Console/Commands/FetchNewArticlesCommand.php
@@ -0,0 +1,22 @@
+<?php
+
+namespace App\Console\Commands;
+
+use App\Services\Articles\ArticleFetcher;
+use Illuminate\Console\Command;
+
+class FetchNewArticlesCommand extends Command
+{
+    protected $signature = 'articles:fetch';
+
+    protected $description = 'Fetches new articles';
+
+    public function handle(): int
+    {
+        logger('Fetch new articles command');
+
+        ArticleFetcher::getNewArticles();
+
+        return self::SUCCESS;
+    }
+}
diff --git a/app/Models/Article.php b/app/Models/Article.php
new file mode 100644
index 0000000..63d2196
--- /dev/null
+++ b/app/Models/Article.php
@@ -0,0 +1,27 @@
+<?php
+
+namespace App\Models;
+
+use Database\Factories\ArticleFactory;
+use Illuminate\Database\Eloquent\Factories\HasFactory;
+use Illuminate\Database\Eloquent\Model;
+
+/**
+ * @method static firstOrCreate(string[] $array)
+ */
+class Article extends Model
+{
+    /** @use HasFactory<ArticleFactory> */
+    use HasFactory;
+
+    protected $fillable = [
+        'url',
+    ];
+
+    public function casts(): array
+    {
+        return [
+            'created_at' => 'datetime',
+        ];
+    }
+}
diff --git a/app/Services/Articles/ArticleFetcher.php b/app/Services/Articles/ArticleFetcher.php
new file mode 100644
index 0000000..ee08eec
--- /dev/null
+++ b/app/Services/Articles/ArticleFetcher.php
@@ -0,0 +1,64 @@
+<?php
+
+namespace App\Services\Articles;
+
+use App\Models\Article;
+use Exception;
+use Illuminate\Database\Eloquent\Collection;
+use Illuminate\Support\Facades\Http;
+
+class ArticleFetcher
+{
+    public static function getNewArticles(): Collection
+    {
+        return self::fetchArticles()
+            ->map(fn (string $url) => self::saveArticle($url));
+    }
+
+    private static function fetchArticles(): Collection
+    {
+        try {
+            $response = Http::get('https://www.vrt.be/vrtnws/en/');
+            $html = $response->body();
+
+            // Extract article links using regex
+            preg_match_all('/href="(\/vrtnws\/en\/\d{4}\/\d{2}\/\d{2}\/[^"]+)"/', $html, $matches);
+
+            $urls = collect($matches[1] ?? [])
+                ->unique()
+                ->take(10) // Limit to 10 articles
+                ->map(fn($path) => 'https://www.vrt.be' . $path)
+                ->toArray();
+
+            $responses = Http::pool(function ($pool) use ($urls) {
+                foreach ($urls as $url) {
+                    $pool->get($url);
+                }
+            });
+
+            return collect($responses)
+                ->map(function ($response, $index) use ($urls) {
+                    $url = $urls[$index];
+
+                    try {
+                        if ($response->successful()) {
+                            return $url;
+                        } else {
+                            return null;
+                        }
+                    } catch (Exception) {
+                        return null;
+                    }
+                })
+                ->filter(fn($article) => !empty($article));
+        } catch (Exception $e) {
+            logger('article_fetcher')->error("Failed to fetch VRT homepage", ['error' => $e->getMessage()]);
+            return new Collection([]);
+        }
+    }
+
+    protected static function saveArticle(string $url): Article
+    {
+        return Article::firstOrCreate(['url' => $url]);
+    }
+}
diff --git a/database/factories/ArticleFactory.php b/database/factories/ArticleFactory.php
new file mode 100644
index 0000000..f62004c
--- /dev/null
+++ b/database/factories/ArticleFactory.php
@@ -0,0 +1,23 @@
+<?php
+
+namespace Database\Factories;
+
+use Illuminate\Database\Eloquent\Factories\Factory;
+
+/**
+ * @extends \Illuminate\Database\Eloquent\Factories\Factory<\App\Models\Article>
+ */
+class ArticleFactory extends Factory
+{
+    /**
+     * Define the model's default state.
+     *
+     * @return array<string, mixed>
+     */
+    public function definition(): array
+    {
+        return [
+            //
+        ];
+    }
+}
diff --git a/database/migrations/2025_06_29_072202_create_articles_table.php b/database/migrations/2025_06_29_072202_create_articles_table.php
new file mode 100644
index 0000000..db3d3a8
--- /dev/null
+++ b/database/migrations/2025_06_29_072202_create_articles_table.php
@@ -0,0 +1,22 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+return new class extends Migration
+{
+    public function up(): void
+    {
+        Schema::create('articles', function (Blueprint $table) {
+            $table->id();
+            $table->string('url');
+            $table->timestamp('created_at')->default(now());
+        });
+    }
+
+    public function down(): void
+    {
+        Schema::dropIfExists('articles');
+    }
+};
diff --git a/docker-compose.yml b/docker-compose.yml
index 6114e5a..fafcec4 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -23,6 +23,25 @@ services:
             - sail
         depends_on:
             - mysql
+    scheduler:
+        build:
+            context: './vendor/laravel/sail/runtimes/8.4'
+            dockerfile: Dockerfile
+            args:
+                WWWGROUP: '${WWWGROUP}'
+        image: 'sail-8.4/app'
+        extra_hosts:
+            - 'host.docker.internal:host-gateway'
+        environment:
+            WWWUSER: '${WWWUSER}'
+            LARAVEL_SAIL: 1
+        volumes:
+            - '.:/var/www/html'
+        networks:
+            - sail
+        depends_on:
+            - mysql
+        command: php artisan schedule:work
     mysql:
         image: 'mysql/mysql-server:8.0'
         ports:
diff --git a/routes/console.php b/routes/console.php
index 3c9adf1..aa1da9f 100644
--- a/routes/console.php
+++ b/routes/console.php
@@ -1,8 +1,6 @@
 <?php
 
-use Illuminate\Foundation\Inspiring;
-use Illuminate\Support\Facades\Artisan;
+use App\Console\Commands\FetchNewArticlesCommand;
+use Illuminate\Support\Facades\Schedule;
 
-Artisan::command('inspire', function () {
-    $this->comment(Inspiring::quote());
-})->purpose('Display an inspiring quote');
+Schedule::command(FetchNewArticlesCommand::class)->hourly();