diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..2b2e134 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,44 @@ +# Development files +node_modules/ +npm-debug.log* +yarn-debug.log* +yarn-error.log* + +# Environment files +.env +.env.* +!.env.production + +# Development tools +.git/ +.gitignore +README.md +docker-compose.yml + +# IDE files +.vscode/ +.idea/ +*.swp +*.swo + +# OS files +.DS_Store +Thumbs.db + +# Laravel development +storage/logs/* +storage/framework/cache/* +storage/framework/sessions/* +storage/framework/testing/* +storage/framework/views/* +bootstrap/cache/* + +# Testing +tests/ +phpunit.xml +.phpunit.result.cache + +# Build artifacts +vendor/ +build/ +dist/ \ No newline at end of file diff --git a/.gitignore b/.gitignore index d9f22cf..3fb54ba 100644 --- a/.gitignore +++ b/.gitignore @@ -9,7 +9,6 @@ /vendor .env .env.backup -.env.production .phpactor.json .phpunit.result.cache Homestead.json diff --git a/Dockerfile b/Dockerfile index ce5bee4..e28b5b1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -35,6 +35,10 @@ COPY . . # Install dependencies RUN composer install --no-dev --optimize-autoloader --no-interaction +# Copy production environment file and generate APP_KEY +COPY docker/.env.production .env +RUN php artisan key:generate + # Copy built frontend assets COPY --from=frontend-builder /app/public/build /var/www/html/public/build diff --git a/README.md b/README.md index 75b4d15..9378dd7 100644 --- a/README.md +++ b/README.md @@ -23,15 +23,9 @@ ### Docker Compose ports: - "8000:8000" environment: - - APP_ENV=production - - APP_KEY=${APP_KEY} - - DB_CONNECTION=mysql - - DB_HOST=mysql - - DB_PORT=3306 - - DB_DATABASE=lemmy_poster - - DB_USERNAME=lemmy_user + - DB_DATABASE=${DB_DATABASE} + - DB_USERNAME=${DB_USERNAME} - DB_PASSWORD=${DB_PASSWORD} - - QUEUE_CONNECTION=database - LEMMY_INSTANCE=${LEMMY_INSTANCE} - LEMMY_USERNAME=${LEMMY_USERNAME} - LEMMY_PASSWORD=${LEMMY_PASSWORD} @@ -46,15 +40,9 @@ ### Docker Compose image: your-registry/lemmy-poster:latest command: ["queue"] environment: - - APP_ENV=production - - APP_KEY=${APP_KEY} - - DB_CONNECTION=mysql - - DB_HOST=mysql - - DB_PORT=3306 - - DB_DATABASE=lemmy_poster - - DB_USERNAME=lemmy_user + - DB_DATABASE=${DB_DATABASE} + - DB_USERNAME=${DB_USERNAME} - DB_PASSWORD=${DB_PASSWORD} - - QUEUE_CONNECTION=database - LEMMY_INSTANCE=${LEMMY_INSTANCE} - LEMMY_USERNAME=${LEMMY_USERNAME} - LEMMY_PASSWORD=${LEMMY_PASSWORD} @@ -68,10 +56,10 @@ ### Docker Compose mysql: image: mysql:8.0 environment: - - MYSQL_DATABASE=lemmy_poster - - MYSQL_USER=lemmy_user + - MYSQL_DATABASE=${DB_DATABASE} + - MYSQL_USER=${DB_USERNAME} - MYSQL_PASSWORD=${DB_PASSWORD} - - MYSQL_ROOT_PASSWORD=${DB_ROOT_PASSWORD} + - MYSQL_ROOT_PASSWORD=${DB_PASSWORD} volumes: - mysql_data:/var/lib/mysql restart: unless-stopped @@ -86,25 +74,38 @@ ### Environment Variables Create a `.env` file with: ```env -APP_KEY=your-app-key-here -DB_PASSWORD=your-db-password -DB_ROOT_PASSWORD=your-root-password +# Database Settings +DB_DATABASE=lemmy_poster +DB_USERNAME=lemmy_user +DB_PASSWORD=your-password + +# Lemmy Settings LEMMY_INSTANCE=your-lemmy-instance.com LEMMY_USERNAME=your-lemmy-username LEMMY_PASSWORD=your-lemmy-password LEMMY_COMMUNITY=your-target-community ``` -Generate the APP_KEY: -```bash -echo "base64:$(openssl rand -base64 32)" -``` - ### Deployment -1. Build and push the image +1. Build and push the image to your registry 2. Copy the docker-compose.yml to your server 3. Create the .env file with your environment variables 4. Run: `docker-compose up -d` +The application will automatically: +- Wait for the database to be ready +- Run database migrations on first startup +- Start the queue worker after migrations complete +- Handle race conditions between web and queue containers + The web interface will be available on port 8000, ready for CloudFlare tunnel configuration. + +### Architecture + +The application uses a multi-container setup: +- **app-web**: Serves the Laravel web interface and handles HTTP requests +- **app-queue**: Processes background jobs (article fetching, Lemmy posting) +- **mysql**: Database storage for articles, logs, and application data + +Both app containers use the same Docker image but with different commands (`web` or `queue`). Environment variables are passed from your `.env` file to configure database access and Lemmy integration. diff --git a/app/Jobs/SyncChannelPostsJob.php b/app/Jobs/SyncChannelPostsJob.php index 66d1316..d787fa0 100644 --- a/app/Jobs/SyncChannelPostsJob.php +++ b/app/Jobs/SyncChannelPostsJob.php @@ -56,21 +56,27 @@ private function syncLemmyChannelPosts(): void private function getAuthToken(LemmyApiService $api): string { - return Cache::remember('lemmy_jwt_token', 3600, function () use ($api) { - $username = config('lemmy.username'); - $password = config('lemmy.password'); + $cachedToken = Cache::get('lemmy_jwt_token'); + + if ($cachedToken) { + return $cachedToken; + } - if (!$username || !$password) { - throw new PlatformAuthException(PlatformEnum::LEMMY, 'Missing credentials'); - } + $username = config('lemmy.username'); + $password = config('lemmy.password'); - $token = $api->login($username, $password); - - if (!$token) { - throw new PlatformAuthException(PlatformEnum::LEMMY, 'Login failed'); - } + if (!$username || !$password) { + throw new PlatformAuthException(PlatformEnum::LEMMY, 'Missing credentials'); + } - return $token; - }); + $token = $api->login($username, $password); + + if (!$token) { + throw new PlatformAuthException(PlatformEnum::LEMMY, 'Login failed'); + } + + Cache::put('lemmy_jwt_token', $token, 3600); + + return $token; } } diff --git a/app/Listeners/LogExceptionToDatabase.php b/app/Listeners/LogExceptionToDatabase.php index 06eb5a7..23da3a2 100644 --- a/app/Listeners/LogExceptionToDatabase.php +++ b/app/Listeners/LogExceptionToDatabase.php @@ -5,12 +5,8 @@ use App\Events\ExceptionLogged; use App\Events\ExceptionOccurred; use App\Models\Log; -use Illuminate\Contracts\Queue\ShouldQueue; -use Illuminate\Queue\InteractsWithQueue; - -class LogExceptionToDatabase implements ShouldQueue +class LogExceptionToDatabase { - use InteractsWithQueue; public function handle(ExceptionOccurred $event): void { diff --git a/app/Modules/Lemmy/Services/LemmyPublisher.php b/app/Modules/Lemmy/Services/LemmyPublisher.php index a8fd87d..ab00a74 100644 --- a/app/Modules/Lemmy/Services/LemmyPublisher.php +++ b/app/Modules/Lemmy/Services/LemmyPublisher.php @@ -58,29 +58,43 @@ public function publish(Article $article, array $extractedData): ArticlePublicat private function getAuthToken(): string { - return Cache::remember('lemmy_jwt_token', 3600, function () { - $username = config('lemmy.username'); - $password = config('lemmy.password'); + $cachedToken = Cache::get('lemmy_jwt_token'); + + if ($cachedToken) { + return $cachedToken; + } - if (!$username || !$password) { - throw new PlatformAuthException(PlatformEnum::LEMMY, 'Missing credentials'); - } + $username = config('lemmy.username'); + $password = config('lemmy.password'); - $token = $this->api->login($username, $password); + if (!$username || !$password) { + throw new PlatformAuthException(PlatformEnum::LEMMY, 'Missing credentials'); + } - if (!$token) { - throw new PlatformAuthException(PlatformEnum::LEMMY, 'Login failed'); - } + $token = $this->api->login($username, $password); - return $token; - }); + if (!$token) { + throw new PlatformAuthException(PlatformEnum::LEMMY, 'Login failed'); + } + + Cache::put('lemmy_jwt_token', $token, 3600); + + return $token; } private function getCommunityId(): int { - return Cache::remember("lemmy_community_id_{$this->community}", 3600, function () { - return $this->api->getCommunityId($this->community); - }); + $cacheKey = "lemmy_community_id_{$this->community}"; + $cachedId = Cache::get($cacheKey); + + if ($cachedId) { + return $cachedId; + } + + $communityId = $this->api->getCommunityId($this->community); + Cache::put($cacheKey, $communityId, 3600); + + return $communityId; } private function createPublicationRecord(Article $article, array $postData, int $communityId): ArticlePublication diff --git a/app/Services/Article/ValidationService.php b/app/Services/Article/ValidationService.php index c2dee91..ffff924 100644 --- a/app/Services/Article/ValidationService.php +++ b/app/Services/Article/ValidationService.php @@ -11,6 +11,21 @@ public static function validate(Article $article): Article logger('Checking keywords for article: ' . $article->id); $articleData = ArticleFetcher::fetchArticleData($article); + + if (!isset($articleData['full_article'])) { + logger()->warning('Article data missing full_article key', [ + 'article_id' => $article->id, + 'url' => $article->url + ]); + + $article->update([ + 'is_valid' => false, + 'validated_at' => now(), + ]); + + return $article->refresh(); + } + $validationResult = self::validateByKeywords($articleData['full_article']); $article->update([ diff --git a/app/Services/Parsers/BelgaHomepageParser.php b/app/Services/Parsers/BelgaHomepageParser.php index 2606222..6a5371d 100644 --- a/app/Services/Parsers/BelgaHomepageParser.php +++ b/app/Services/Parsers/BelgaHomepageParser.php @@ -6,11 +6,10 @@ class BelgaHomepageParser { public static function extractArticleUrls(string $html): array { - preg_match_all('/href="https:\/\/www\.belganewsagency\.eu\/([a-z0-9-]+)"/', $html, $matches); + preg_match_all('/href="(https:\/\/www\.belganewsagency\.eu\/[a-z0-9-]+)"/', $html, $matches); - $urls = collect($matches[0] ?? []) + $urls = collect($matches[1] ?? []) ->unique() - ->map(fn ($url) => str_replace('href="', '', str_replace('"', '', $url))) ->toArray(); return $urls; diff --git a/docker/.env.production b/docker/.env.production new file mode 100644 index 0000000..952999e --- /dev/null +++ b/docker/.env.production @@ -0,0 +1,59 @@ +APP_NAME="Lemmy Poster" +APP_ENV=production +APP_KEY= +APP_DEBUG=false +APP_URL=http://localhost + +APP_LOCALE=en +APP_FALLBACK_LOCALE=en +APP_FAKER_LOCALE=en_US + +APP_MAINTENANCE_DRIVER=file + +PHP_CLI_SERVER_WORKERS=4 + +BCRYPT_ROUNDS=12 + +LOG_CHANNEL=stack +LOG_STACK=single +LOG_DEPRECATIONS_CHANNEL=null +LOG_LEVEL=error + +DB_CONNECTION=mysql +DB_HOST=mysql +DB_PORT=3306 +DB_DATABASE=lemmy_poster +DB_USERNAME=lemmy_user +DB_PASSWORD=your-password + +SESSION_DRIVER=database +SESSION_LIFETIME=120 +SESSION_ENCRYPT=false +SESSION_PATH=/ +SESSION_DOMAIN=null + +BROADCAST_CONNECTION=log +FILESYSTEM_DISK=local +QUEUE_CONNECTION=database + +CACHE_STORE=database + +REDIS_CLIENT=phpredis +REDIS_HOST=127.0.0.1 +REDIS_PASSWORD=null +REDIS_PORT=6379 + +MAIL_MAILER=log +MAIL_SCHEME=null +MAIL_HOST=127.0.0.1 +MAIL_PORT=2525 +MAIL_USERNAME=null +MAIL_PASSWORD=null +MAIL_FROM_ADDRESS="hello@example.com" +MAIL_FROM_NAME="${APP_NAME}" + +# LEMMY SETTINGS +LEMMY_INSTANCE= +LEMMY_USERNAME= +LEMMY_PASSWORD= +LEMMY_COMMUNITY= diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index f37a38c..de00dd8 100644 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -15,14 +15,35 @@ if [ -z "$LEMMY_INSTANCE" ] || [ -z "$LEMMY_USERNAME" ] || [ -z "$LEMMY_PASSWORD fi # Wait for database to be ready -until php artisan tinker --execute="DB::connection()->getPdo();" > /dev/null 2>&1; do - echo "Waiting for database connection..." - sleep 2 +echo "Waiting for database connection..." +until php -r " +try { + \$pdo = new PDO('mysql:host=mysql;port=3306;dbname=' . getenv('DB_DATABASE'), getenv('DB_USERNAME'), getenv('DB_PASSWORD')); + echo 'Connected'; + exit(0); +} catch (Exception \$e) { + exit(1); +} +" > /dev/null 2>&1; do + echo "Database not ready, waiting..." + sleep 5 done +echo "Database connection established." # Run migrations on first start (web container only) if [ "$1" = "web" ]; then + echo "Running database migrations..." php artisan migrate --force +elif [ "$1" = "queue" ]; then + echo "Waiting for migrations to complete..." + # Wait for all migrations to actually finish running + until php artisan migrate:status 2>/dev/null | grep -c "Ran" | grep -q "7"; do + echo "Migrations still running, waiting..." + sleep 3 + done + echo "All migrations completed." + echo "Waiting for database to stabilize..." + sleep 5 fi # Execute the command based on the argument diff --git a/routes/console.php b/routes/console.php index bc6a0cc..db0d635 100644 --- a/routes/console.php +++ b/routes/console.php @@ -37,7 +37,7 @@ } else { logger()->debug('No unpublished valid articles found for Lemmy publishing'); } -})->everyFifteenMinutes()->name('publish-to-lemmy'); +})->everyFiveMinutes()->name('publish-to-lemmy'); Schedule::call(function () { $communityId = config('lemmy.community_id');