cache chatlib databases

diff --git a/app/Console/Commands/ChatlibDatabase.php b/app/Console/Commands/ChatlibDatabase.php
index bfe0135c4b8e00e684a9f2974faf862f447296fd..b95bb64a06a6dbec47b333c2c9248493e0c8492d 100644 (file)
--- a/app/Console/Commands/ChatlibDatabase.php
+++ b/app/Console/Commands/ChatlibDatabase.php
@@ -34,6 +34,7 @@ class ChatlibDatabase extends Command {
                ChatLog::where('type', '=', 'chat')
                        ->where('banned', '=', false)
                        ->whereNotNull('evaluated_at')
+                       ->where('created_at', '<', now()->sub(7, 'day'))
                        ->chunk(5000, function ($msgs) use ($de, $en) {
                                foreach ($msgs as $msg) {
                                        if ($msg->detected_language === 'de') {
@@ -48,16 +49,10 @@ class ChatlibDatabase extends Command {
                        });
 
                $de->compile();
-               $en->compile();
+               $de->saveAs('de');
 
-               $this->line('');
-               for ($i = 0; $i < 50; ++$i) {
-                       $this->line($de->generate());
-               }
-               $this->line('');
-               for ($i = 0; $i < 50; ++$i) {
-                       $this->line($en->generate());
-               }
+               $en->compile();
+               $en->saveAs('en');
 
                return 0;
        }

diff --git a/app/Console/Commands/ChatlibGenerate.php b/app/Console/Commands/ChatlibGenerate.php
new file mode 100644 (file)
index 0000000..5ea85f6
--- /dev/null
+++ b/app/Console/Commands/ChatlibGenerate.php
@@ -0,0 +1,43 @@
+<?php
+
+namespace App\Console\Commands;
+
+use App\Models\ChatLib;
+use Illuminate\Console\Command;
+
+class ChatlibGenerate extends Command {
+
+       /**
+        * The name and signature of the console command.
+        *
+        * @var string
+        */
+       protected $signature = 'chatlib:generate {which=de} {amount=50}';
+
+       /**
+        * The console command description.
+        *
+        * @var string
+        */
+       protected $description = 'Generates samples from a ChatLib database';
+
+       /**
+        * Execute the console command.
+        *
+        * @return int
+        */
+       public function handle() {
+               $db = new ChatLib();
+               $db->loadFrom($this->argument('which'));
+
+               $amount = intval($this->argument('amount'));
+               for ($i = 0; $i < $amount; ++$i) {
+                       $this->line($db->generate());
+               }
+
+               return 0;
+       }
+
+}
+
+?>

diff --git a/app/Models/ChatLib.php b/app/Models/ChatLib.php
index 417b18f3dd6b01f1cbeb5b923d71148ebb995cd3..cf5d7dd6c825f6d8f37451dbfe7136bd0c91b787 100644 (file)
--- a/app/Models/ChatLib.php
+++ b/app/Models/ChatLib.php
@@ -2,6 +2,8 @@
 
 namespace App\Models;
 
+use Illuminate\Support\Facades\Storage;
+
 class ChatLib {
 
        public function addMessage($msg) {
@@ -29,7 +31,6 @@ class ChatLib {
                                unset($this->transitions[$key]);
                        }
                }
-               echo 'size: ', number_format(strlen(json_encode($this->transitions)), 0), PHP_EOL;
        }
 
        public function generate($limit = 100) {
@@ -44,6 +45,20 @@ class ChatLib {
                return $generated;
        }
 
+       public function saveAs($name) {
+               $data = [
+                       'size' => $this->size,
+                       'transitions' => $this->transitions,
+               ];
+               Storage::disk('chatlib')->put($name.'.json', json_encode($data));
+       }
+
+       public function loadFrom($name) {
+               $data = json_decode(Storage::disk('chatlib')->get($name.'.json'), true);
+               $this->size = $data['size'];
+               $this->transitions = $data['transitions'];
+       }
+
        private function index($arr) {
                $result = [];
                $sum = 0;
@@ -132,7 +147,11 @@ class ChatLib {
        }
 
        private function tokenize($str) {
-               return array_values(array_filter(preg_split('/\b/u', $str)));
+               return array_values(array_filter(preg_split('/\b/u', $str), function($token) {
+                       if (empty($token)) return false;
+                       if (preg_match('/cheer\d+/u', strtolower($token))) return false;
+                       return true;
+               }));
        }
 
        private function generalize($tokens) {

diff --git a/config/filesystems.php b/config/filesystems.php
index d1e5c6c453c72ebd2a8307f6c37efa285e9ffa32..bfcbf29f88040ee970c2d0c9908aa8ae785194ce 100644 (file)
--- a/config/filesystems.php
+++ b/config/filesystems.php
@@ -56,6 +56,11 @@ return [
                        'root' => storage_path('app/alttp-spoilers'),
                ],
 
+               'chatlib' => [
+                       'driver' => 'local',
+                       'root' => storage_path('app/chatlib'),
+               ],
+
                'media' => [
                        'driver' => 'local',
                        'root' => storage_path('app/media'),