separate chatlib database generation

[alttp.git] / app / Console / Commands / ChatlibDatabase.php

diff --git a/app/Console/Commands/ChatlibDatabase.php b/app/Console/Commands/ChatlibDatabase.php
index b701ad144b3d0aed3abc26cb481813e4347b0b4e..e492bb7c1169160fafea1c9a938aec7f5b0c8b86 100644 (file)
--- a/app/Console/Commands/ChatlibDatabase.php
+++ b/app/Console/Commands/ChatlibDatabase.php
@@ -13,14 +13,14 @@ class ChatlibDatabase extends Command {
         *
         * @var string
         */
-       protected $signature = 'chatlib:database';
+       protected $signature = 'chatlib:database {which=de} {size=7}';
 
        /**
         * The console command description.
         *
         * @var string
         */
-       protected $description = 'Updates the ChatLib database';
+       protected $description = 'Update a ChatLib database';
 
        /**
         * Execute the console command.
@@ -29,36 +29,37 @@ class ChatlibDatabase extends Command {
         */
        public function handle() {
                $count = 0;
+               $start = time();
 
-               $de = new ChatLib();
-               $en = new ChatLib();
+               $size = $this->argument('size');
+               $lang = $this->argument('which');
+               $db = new ChatLib($size);
 
                ChatLog::where('type', '=', 'chat')
                        ->where('banned', '=', false)
                        ->whereNotNull('evaluated_at')
                        ->where('created_at', '<', now()->sub(7, 'day'))
                        ->whereNotIn('classification', ['gg', 'gl', 'number', 'o7'])
-                       ->whereRaw('LENGTH(`text_content`) > 12')
-                       ->chunk(5000, function ($msgs) use (&$count, $de, $en) {
+                       ->where(function ($query) use ($lang) {
+                               $query->whereNull('detected_language');
+                               $query->orWhere('detected_language', '=', $lang);
+                       })
+                       ->whereRaw('LENGTH(`text_content`) > 10')
+                       ->chunk(5000, function ($msgs) use (&$count, $db) {
                                foreach ($msgs as $msg) {
-                                       if ($msg->detected_language === 'de') {
-                                               $de->addMessage($msg);
-                                       } else if ($msg->detected_language === 'en') {
-                                               $en->addMessage($msg);
-                                       } else if (is_null($msg->detected_language)) {
-                                               $de->addMessage($msg);
-                                               $en->addMessage($msg);
-                                       }
+                                       $db->addMessage($msg);
                                        ++$count;
                                }
                                $this->line($count);
                        });
 
-               $de->compile();
-               $de->saveAs('de');
+               $db->compile();
+               $db->saveAs($lang);
 
-               $en->compile();
-               $en->saveAs('en');
+               $this->line(
+                       number_format(time() - $start, 0).'s '.
+                       number_format(memory_get_usage() / 1024 / 1024, 3).'MB now '.
+                       number_format(memory_get_peak_usage() / 1024 / 1024, 3).'MB peak');
 
                return 0;
        }