*
* @var string
*/
- protected $signature = 'chatlib:database';
+ protected $signature = 'chatlib:database {which=de} {size=7}';
/**
* The console command description.
*
* @var string
*/
- protected $description = 'Updates the ChatLib database';
+ protected $description = 'Update a ChatLib database';
/**
* Execute the console command.
* @return int
*/
public function handle() {
- $de = new ChatLib();
- $en = new ChatLib();
+ $count = 0;
+ $start = time();
+
+ $size = $this->argument('size');
+ $lang = $this->argument('which');
+ $db = new ChatLib($size);
ChatLog::where('type', '=', 'chat')
->where('banned', '=', false)
->whereNotNull('evaluated_at')
->where('created_at', '<', now()->sub(7, 'day'))
- ->chunk(5000, function ($msgs) use ($de, $en) {
+ ->whereNotIn('classification', ['gg', 'gl', 'number', 'o7'])
+ ->where(function ($query) use ($lang) {
+ $query->whereNull('detected_language');
+ $query->orWhere('detected_language', '=', $lang);
+ })
+ ->whereRaw('LENGTH(`text_content`) > 10')
+ ->chunk(5000, function ($msgs) use (&$count, $db) {
foreach ($msgs as $msg) {
- if ($msg->detected_language === 'de') {
- $de->addMessage($msg);
- } else if ($msg->detected_language === 'en') {
- $en->addMessage($msg);
- } else if (is_null($msg->detected_language)) {
- $de->addMessage($msg);
- $en->addMessage($msg);
- }
+ $db->addMessage($msg);
+ ++$count;
}
+ $this->line($count);
});
- $de->compile();
- $de->saveAs('de');
+ $db->compile();
+ $db->saveAs($lang);
- $en->compile();
- $en->saveAs('en');
+ $this->line(
+ number_format(time() - $start, 0).'s '.
+ number_format(memory_get_usage() / 1024 / 1024, 3).'MB now '.
+ number_format(memory_get_peak_usage() / 1024 / 1024, 3).'MB peak');
return 0;
}