From a45648fa8ecf7712c7fd00eb2b93e862b5264f04 Mon Sep 17 00:00:00 2001
From: Daniel Karbach <daniel.karbach@localhorst.tv>
Date: Wed, 8 May 2024 13:59:22 +0200
Subject: [PATCH] cache chatlib databases

---
 app/Console/Commands/ChatlibDatabase.php | 13 +++----
 app/Console/Commands/ChatlibGenerate.php | 43 ++++++++++++++++++++++++
 app/Models/ChatLib.php                   | 23 +++++++++++--
 config/filesystems.php                   |  5 +++
 4 files changed, 73 insertions(+), 11 deletions(-)
 create mode 100644 app/Console/Commands/ChatlibGenerate.php

diff --git a/app/Console/Commands/ChatlibDatabase.php b/app/Console/Commands/ChatlibDatabase.php
index bfe0135..b95bb64 100644
--- a/app/Console/Commands/ChatlibDatabase.php
+++ b/app/Console/Commands/ChatlibDatabase.php
@@ -34,6 +34,7 @@ class ChatlibDatabase extends Command {
 		ChatLog::where('type', '=', 'chat')
 			->where('banned', '=', false)
 			->whereNotNull('evaluated_at')
+			->where('created_at', '<', now()->sub(7, 'day'))
 			->chunk(5000, function ($msgs) use ($de, $en) {
 				foreach ($msgs as $msg) {
 					if ($msg->detected_language === 'de') {
@@ -48,16 +49,10 @@ class ChatlibDatabase extends Command {
 			});
 
 		$de->compile();
-		$en->compile();
+		$de->saveAs('de');
 
-		$this->line('');
-		for ($i = 0; $i < 50; ++$i) {
-			$this->line($de->generate());
-		}
-		$this->line('');
-		for ($i = 0; $i < 50; ++$i) {
-			$this->line($en->generate());
-		}
+		$en->compile();
+		$en->saveAs('en');
 
 		return 0;
 	}
diff --git a/app/Console/Commands/ChatlibGenerate.php b/app/Console/Commands/ChatlibGenerate.php
new file mode 100644
index 0000000..5ea85f6
--- /dev/null
+++ b/app/Console/Commands/ChatlibGenerate.php
@@ -0,0 +1,43 @@
+<?php
+
+namespace App\Console\Commands;
+
+use App\Models\ChatLib;
+use Illuminate\Console\Command;
+
+class ChatlibGenerate extends Command {
+
+	/**
+	 * The name and signature of the console command.
+	 *
+	 * @var string
+	 */
+	protected $signature = 'chatlib:generate {which=de} {amount=50}';
+
+	/**
+	 * The console command description.
+	 *
+	 * @var string
+	 */
+	protected $description = 'Generates samples from a ChatLib database';
+
+	/**
+	 * Execute the console command.
+	 *
+	 * @return int
+	 */
+	public function handle() {
+		$db = new ChatLib();
+		$db->loadFrom($this->argument('which'));
+
+		$amount = intval($this->argument('amount'));
+		for ($i = 0; $i < $amount; ++$i) {
+			$this->line($db->generate());
+		}
+
+		return 0;
+	}
+
+}
+
+?>
diff --git a/app/Models/ChatLib.php b/app/Models/ChatLib.php
index 417b18f..cf5d7dd 100644
--- a/app/Models/ChatLib.php
+++ b/app/Models/ChatLib.php
@@ -2,6 +2,8 @@
 
 namespace App\Models;
 
+use Illuminate\Support\Facades\Storage;
+
 class ChatLib {
 
 	public function addMessage($msg) {
@@ -29,7 +31,6 @@ class ChatLib {
 				unset($this->transitions[$key]);
 			}
 		}
-		echo 'size: ', number_format(strlen(json_encode($this->transitions)), 0), PHP_EOL;
 	}
 
 	public function generate($limit = 100) {
@@ -44,6 +45,20 @@ class ChatLib {
 		return $generated;
 	}
 
+	public function saveAs($name) {
+		$data = [
+			'size' => $this->size,
+			'transitions' => $this->transitions,
+		];
+		Storage::disk('chatlib')->put($name.'.json', json_encode($data));
+	}
+
+	public function loadFrom($name) {
+		$data = json_decode(Storage::disk('chatlib')->get($name.'.json'), true);
+		$this->size = $data['size'];
+		$this->transitions = $data['transitions'];
+	}
+
 	private function index($arr) {
 		$result = [];
 		$sum = 0;
@@ -132,7 +147,11 @@ class ChatLib {
 	}
 
 	private function tokenize($str) {
-		return array_values(array_filter(preg_split('/\b/u', $str)));
+		return array_values(array_filter(preg_split('/\b/u', $str), function($token) {
+			if (empty($token)) return false;
+			if (preg_match('/cheer\d+/u', strtolower($token))) return false;
+			return true;
+		}));
 	}
 
 	private function generalize($tokens) {
diff --git a/config/filesystems.php b/config/filesystems.php
index d1e5c6c..bfcbf29 100644
--- a/config/filesystems.php
+++ b/config/filesystems.php
@@ -56,6 +56,11 @@ return [
 			'root' => storage_path('app/alttp-spoilers'),
 		],
 
+		'chatlib' => [
+			'driver' => 'local',
+			'root' => storage_path('app/chatlib'),
+		],
+
 		'media' => [
 			'driver' => 'local',
 			'root' => storage_path('app/media'),
-- 
2.39.2