From 792e4d7f289698438414cb9ed3001c41ff3115c2 Mon Sep 17 00:00:00 2001 From: Ilya Rogozhin Date: Tue, 9 Jun 2026 18:52:21 +0200 Subject: [PATCH] Export users 500k+ lines, seed, supervisord setup --- images/php85/Dockerfile | 10 +- images/php85/opcache.ini | 3 +- images/php85/supervisord.conf | 24 + main-repo/README.md | 17 +- .../Http/Controllers/UserExportController.php | 53 +++ main-repo/app/Jobs/UsersExport.php | 63 +++ main-repo/database/factories/UserFactory.php | 2 + ...6_09_143727_phone_and_surname_to_users.php | 30 ++ main-repo/database/seeders/DatabaseSeeder.php | 7 +- main-repo/database/seeders/UserSeeder.php | 31 ++ main-repo/public/exports/.gitignore | 2 + main-repo/resources/views/welcome.blade.php | 440 +++++++++--------- main-repo/routes/web.php | 7 + main-repo/supervisord.pid | 1 + nginx-hosts/test-fhr.ldev.conf | 2 +- 15 files changed, 469 insertions(+), 223 deletions(-) create mode 100644 images/php85/supervisord.conf create mode 100644 main-repo/app/Http/Controllers/UserExportController.php create mode 100644 main-repo/app/Jobs/UsersExport.php create mode 100644 main-repo/database/migrations/2026_06_09_143727_phone_and_surname_to_users.php create mode 100644 main-repo/database/seeders/UserSeeder.php create mode 100644 main-repo/public/exports/.gitignore create mode 100644 main-repo/supervisord.pid diff --git a/images/php85/Dockerfile b/images/php85/Dockerfile index 74824c4..d7cc691 100644 --- a/images/php85/Dockerfile +++ b/images/php85/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ libzip-dev \ unzip \ git \ + supervisor \ && rm -rf /var/lib/apt/lists/* # Configure and install PHP extensions @@ -55,7 +56,14 @@ USER www-data # 6. Download and unpack the Laravel installer directly into www-data's profile folder RUN composer global require laravel/installer --no-interaction +# 7. Switch back to root user +USER root + +# Supervisor +RUN mkdir -p /var/log/supervisor +COPY ./supervisord.conf /etc/supervisor/conf.d/supervisord.conf + # Expose PHP-FPM default communication port EXPOSE 9000 -CMD ["php-fpm"] +CMD ["/usr/bin/supervisord", "-n", "-c", "/etc/supervisor/conf.d/supervisord.conf"] \ No newline at end of file diff --git a/images/php85/opcache.ini b/images/php85/opcache.ini index bb2a2a5..d2945d4 100644 --- a/images/php85/opcache.ini +++ b/images/php85/opcache.ini @@ -4,5 +4,6 @@ opcache.memory_consumption=256 opcache.interned_strings_buffer=16 opcache.max_accelerated_files=20000 opcache.revalidate_freq=0 -opcache.validate_timestamps=0 +opcache.validate_timestamps=1 +opcachee.revalidate_freq=0 opcache.fast_shutdown=1 diff --git a/images/php85/supervisord.conf b/images/php85/supervisord.conf new file mode 100644 index 0000000..af90683 --- /dev/null +++ b/images/php85/supervisord.conf @@ -0,0 +1,24 @@ +[supervisord] +nodaemon=true +user=root + +[program:php-fpm] +command=php-fpm +autostart=true +autorestart=true +priority=5 +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 + +[program:laravel-worker] +process_name=%(program_name)s_%(process_num)02d +command=php /var/www/test-fhr/artisan queue:work --sleep=3 --tries=3 +autostart=true +autorestart=true +user=www-data +numprocs=4 +redirect_stderr=true +stdout_logfile=/var/www/test-fhr/storage/logs/worker.log +stopwaitsecs=3600 \ No newline at end of file diff --git a/main-repo/README.md b/main-repo/README.md index 42b65de..a463b84 100644 --- a/main-repo/README.md +++ b/main-repo/README.md @@ -30,7 +30,7 @@ docker compose up alias fhr-mysql='docker exec -it mysql_fhr /bin/bash' alias fhr-nginx='docker exec -it nginx_fhr /bin/bash' alias fhr-redis='docker exec -it redis_fhr /bin/sh' -alias fhr-php='docker exec -it php8_fhr /bin/bash' +alias fhr-php='docker exec -u www-data:www-data -it php8_fhr /bin/bash' ``` ### Написать сортировку для массива числовых данных от 200 тысяч элементов @@ -74,4 +74,17 @@ mysqldump -u root -psecret test_fhr clubs players > clubs_and_players_dump.sql После такой выгрузки, дамп появится в папке MySQL-files Тестовый дамп лежит в корне проекта **clubs_and_players_dump.sql** -**Время: 1 час 25 минут.** \ No newline at end of file +**Время: 1 час 25 минут.** + +### Выгрузка БД пользователей, более 500к+ строк + +Сидируем БД +```bash +# Долгий сид достаточно, минут 15-20 возможно +php artisan db:seed --class=UserSeeder +``` + +Дальше открываем главную страницу http://test-fhr.ldev/ +Хост **test-fhr.ldev** нужно будет прописать в hosts в зависимости от вашей ОС. + +**Время: 2 часа 30 минут.** \ No newline at end of file diff --git a/main-repo/app/Http/Controllers/UserExportController.php b/main-repo/app/Http/Controllers/UserExportController.php new file mode 100644 index 0000000..5b1ca97 --- /dev/null +++ b/main-repo/app/Http/Controllers/UserExportController.php @@ -0,0 +1,53 @@ +addMinutes(2), function () { + return 'standby'; + }); + + $lastExportFile = Cache::get('last_export_file'); + + return [ + 'export_status' => $status, + 'last_export_file' => $lastExportFile + ]; + } + + public function counter() + { + $currentCount = Cache::remember('users_count', now()->addMinutes(2), function () { + return User::count(); + }); + + return [ + 'count' => $currentCount, + ]; + } + + public function export() + { + $status = cache('export_status'); + if ($status === 'progress') { + return [ + 'export_status' => $status, + ]; + } + + Cache::forever('export_status', 'progress'); + + UsersExport::dispatch(); + + return [ + 'export_status' => 'started', + ]; + } +} \ No newline at end of file diff --git a/main-repo/app/Jobs/UsersExport.php b/main-repo/app/Jobs/UsersExport.php new file mode 100644 index 0000000..a4f47e9 --- /dev/null +++ b/main-repo/app/Jobs/UsersExport.php @@ -0,0 +1,63 @@ +format('Y-m-d_H-i-s') . '.csv'; + $filePath = $exportPath . '/' . $filename; + + $handle = fopen($filePath, 'w'); + + fputcsv($handle, ['Имя', 'Фамилия', 'Телефон', 'E-mail']); + + foreach (User::select('name', 'surname', 'phone_number', 'email')->lazy() as $user) { + fputcsv($handle, [ + $user->name, + $user->surname, + $user->phone_number, + $user->email, + ]); + } + + fclose($handle); + + Cache::forever('export_status', 'standby'); + Cache::forever('last_export_file', 'exports/' . $filename); + + } catch (Throwable $e) { + Cache::forever('export_status', 'failed'); + report($e); + throw $e; + } + } +} diff --git a/main-repo/database/factories/UserFactory.php b/main-repo/database/factories/UserFactory.php index c4ceb07..1e99974 100644 --- a/main-repo/database/factories/UserFactory.php +++ b/main-repo/database/factories/UserFactory.php @@ -26,6 +26,8 @@ class UserFactory extends Factory { return [ 'name' => fake()->name(), + 'surname' => fake()->lastName(), + 'phone_number' => fake()->phoneNumber(), 'email' => fake()->unique()->safeEmail(), 'email_verified_at' => now(), 'password' => static::$password ??= Hash::make('password'), diff --git a/main-repo/database/migrations/2026_06_09_143727_phone_and_surname_to_users.php b/main-repo/database/migrations/2026_06_09_143727_phone_and_surname_to_users.php new file mode 100644 index 0000000..df16049 --- /dev/null +++ b/main-repo/database/migrations/2026_06_09_143727_phone_and_surname_to_users.php @@ -0,0 +1,30 @@ +string('surname')->after('name'); + $table->string('phone_number')->after('surname'); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('users', function (Blueprint $table) { + $table->dropColumn('surname'); + $table->dropColumn('phone_number'); + }); + } +}; diff --git a/main-repo/database/seeders/DatabaseSeeder.php b/main-repo/database/seeders/DatabaseSeeder.php index 6b901f8..34dd1c0 100644 --- a/main-repo/database/seeders/DatabaseSeeder.php +++ b/main-repo/database/seeders/DatabaseSeeder.php @@ -15,11 +15,6 @@ class DatabaseSeeder extends Seeder */ public function run(): void { - // User::factory(10)->create(); - - User::factory()->create([ - 'name' => 'Test User', - 'email' => 'test@example.com', - ]); + // } } diff --git a/main-repo/database/seeders/UserSeeder.php b/main-repo/database/seeders/UserSeeder.php new file mode 100644 index 0000000..01f7451 --- /dev/null +++ b/main-repo/database/seeders/UserSeeder.php @@ -0,0 +1,31 @@ +command->getOutput()->progressStart($totalToCreate); + + for ($i = 0; $i < $totalToCreate; $i += $chunkSize) { + $createCount = min($chunkSize, $totalToCreate - $i); + User::factory($createCount)->create(); + $this->command->getOutput()->progressAdvance($createCount); + } + + $this->command->getOutput()->progressFinish(); + } +} diff --git a/main-repo/public/exports/.gitignore b/main-repo/public/exports/.gitignore new file mode 100644 index 0000000..d6b7ef3 --- /dev/null +++ b/main-repo/public/exports/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/main-repo/resources/views/welcome.blade.php b/main-repo/resources/views/welcome.blade.php index 26e294a..1278da8 100644 --- a/main-repo/resources/views/welcome.blade.php +++ b/main-repo/resources/views/welcome.blade.php @@ -1,223 +1,239 @@ - - - + + + + Выгрузка пользователей + + + + - {{ config('app.name', 'Laravel') }} +
+

Выгрузка пользователей

- @fonts +
+ Сейчас пользователей в БД: @{{ userCount }} +
- - @if (file_exists(public_path('build/manifest.json')) || file_exists(public_path('hot'))) - @vite(['resources/css/app.css', 'resources/js/app.js']) - @else - - @endif - - -
- @if (Route::has('login')) - - @endif -
-
-
-
-

Let's get started

-

With so many options available to you,
we suggest you start with the following:

- - +
+ Скачать в формате .csv +
-

- v{{ app()->version() }} - - View changelog - - - - -

-
-
- {{-- Laravel Logo --}} - - - - - - - - - +
+ Статус: @{{ statusMessage }} +
+
- {{-- 13 --}} - - - - - - - - - - - - + + -
-
- -
+ createApp({ + setup() { + const userCount = ref('загрузка...'); + const statusMessage = ref('ожидание'); + const isExporting = ref(false); + const downloadUrl = ref(null); + let pollInterval = null; + const csrfToken = document.querySelector('meta[name="csrf-token"]').getAttribute('content'); - @if (Route::has('login')) - - @endif - + const fetchUserCount = async () => { + try { + const response = await fetch('/users-export/counter'); + if (!response.ok) throw new Error('Network response was not ok'); + const data = await response.json(); + userCount.value = data.count.toLocaleString('ru-RU'); + } catch (error) { + console.error('Ошибка при получении количества пользователей:', error); + userCount.value = 'ошибка'; + } + }; + + const startExport = async () => { + try { + isExporting.value = true; + statusMessage.value = 'Отправка запроса...'; + downloadUrl.value = null; + + const response = await fetch('/users-export/export', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'X-CSRF-TOKEN': csrfToken + }, + body: JSON.stringify({}) + }); + + if (!response.ok) { + if (response.status === 419) { + statusMessage.value = 'Ошибка: Сессия истекла. Пожалуйста, обновите страницу.'; + } else { + throw new Error('Network response was not ok: ' + response.statusText); + } + isExporting.value = false; + return; + } + + const data = await response.json(); + + if (data.export_status === 'progress') { + statusMessage.value = 'Выгрузка уже идет. Пока нельзя начать новую выгрузку'; + startPolling(); + } else if (data.export_status === 'started') { + statusMessage.value = 'Выгрузка начата.'; + startPolling(); + } else { + isExporting.value = false; + statusMessage.value = 'Неизвестный статус: ' + data.export_status; + } + + } catch (error) { + console.error('Ошибка при запуске экспорта:', error); + statusMessage.value = 'Ошибка при запуске выгрузки'; + isExporting.value = false; + } + }; + + const startPolling = () => { + if (pollInterval) return; + + pollInterval = setInterval(async () => { + try { + const response = await fetch('/users-export/status'); + if (!response.ok) throw new Error('Network response was not ok'); + const data = await response.json(); + + if (data.export_status === 'standby') { + stopPolling(); + statusMessage.value = 'выгрузка завершена'; + isExporting.value = false; + + if (data.last_export_file) { + downloadUrl.value = '/' + data.last_export_file; + } + } else if (data.export_status === 'failed') { + stopPolling(); + statusMessage.value = 'Ошибка выгрузки (failed)'; + isExporting.value = false; + } + + } catch (error) { + console.error('Ошибка при опросе статуса:', error); + stopPolling(); + statusMessage.value = 'Ошибка связи с сервером при опросе статуса'; + isExporting.value = false; + } + }, 6000); // 6 сек + }; + + const stopPolling = () => { + if (pollInterval) { + clearInterval(pollInterval); + pollInterval = null; + } + }; + + onMounted(() => { + fetchUserCount(); + + fetch('/users-export/status') + .then(res => res.json()) + .then(data => { + if (data.export_status === 'progress') { + isExporting.value = true; + statusMessage.value = 'Выгрузка в процессе...'; + startPolling(); + } else if (data.export_status === 'standby' && data.last_export_file) { + statusMessage.value = 'Готов файл с прошлой выгрузки'; + downloadUrl.value = '/' + data.last_export_file; + } + }) + .catch(err => console.error(err)); + }); + + return { + userCount, + statusMessage, + isExporting, + downloadUrl, + startExport + }; + }, + compilerOptions: { + delimiters: ['@{{', '}}'] + } + }).mount('#app'); + + + diff --git a/main-repo/routes/web.php b/main-repo/routes/web.php index 86a06c5..1078f25 100644 --- a/main-repo/routes/web.php +++ b/main-repo/routes/web.php @@ -1,7 +1,14 @@ group(function () { + Route::get('/counter', [UserExportController::class, 'counter']); + Route::get('/status', [UserExportController::class, 'status']); + Route::post('/export', [UserExportController::class, 'export']); +}); diff --git a/main-repo/supervisord.pid b/main-repo/supervisord.pid new file mode 100644 index 0000000..d00491f --- /dev/null +++ b/main-repo/supervisord.pid @@ -0,0 +1 @@ +1 diff --git a/nginx-hosts/test-fhr.ldev.conf b/nginx-hosts/test-fhr.ldev.conf index 8469628..828e74b 100644 --- a/nginx-hosts/test-fhr.ldev.conf +++ b/nginx-hosts/test-fhr.ldev.conf @@ -44,6 +44,6 @@ server { } location / { - try_files $uri /index.php?$args; + try_files $uri $uri/ /index.php?$query_string; } }