]> git.localhorst.tv Git - alttp.git/blob - app/Console/Commands/ChatlibDatabase.php
revamp chatlib tokenization
[alttp.git] / app / Console / Commands / ChatlibDatabase.php
1 <?php
2
3 namespace App\Console\Commands;
4
5 use App\Models\ChatLib;
6 use App\Models\ChatLog;
7 use Illuminate\Console\Command;
8
9 class ChatlibDatabase extends Command {
10
11         /**
12          * The name and signature of the console command.
13          *
14          * @var string
15          */
16         protected $signature = 'chatlib:database {which=de} {size=3}';
17
18         /**
19          * The console command description.
20          *
21          * @var string
22          */
23         protected $description = 'Update a ChatLib database';
24
25         /**
26          * Execute the console command.
27          *
28          * @return int
29          */
30         public function handle() {
31                 $count = 0;
32                 $start = time();
33
34                 $size = $this->argument('size');
35                 $lang = $this->argument('which');
36                 $db = new ChatLib($size);
37
38                 ChatLog::where('type', '=', 'chat')
39                         ->where('banned', '=', false)
40                         ->whereNotNull('evaluated_at')
41                         ->where('created_at', '<', now()->sub(7, 'day'))
42                         ->whereNotIn('classification', ['gg', 'gl', 'number', 'o7'])
43                         ->where(function ($query) use ($lang) {
44                                 $query->whereNull('detected_language');
45                                 $query->orWhere('detected_language', '=', $lang);
46                         })
47                         ->orderBy('channel_id')
48                         ->orderBy('created_at')
49                         ->chunk(5000, function ($msgs) use (&$count, $db) {
50                                 $previous = null;
51                                 foreach ($msgs as $msg) {
52                                         $db->addMessage($msg, $previous);
53                                         $previous = $msg;
54                                         ++$count;
55                                 }
56                                 $this->line($count);
57                         });
58
59                 $db->compile();
60                 $db->saveAs($lang);
61
62                 $this->line(
63                         number_format(time() - $start, 0).'s '.
64                         number_format(memory_get_usage() / 1024 / 1024, 3).'MB now '.
65                         number_format(memory_get_peak_usage() / 1024 / 1024, 3).'MB peak');
66
67                 return 0;
68         }
69
70 }
71
72 ?>